From 57193ca9746bab8b3db85e75cf219cb744679cb9 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Wed, 23 Jul 2025 11:32:57 -0700 Subject: [PATCH 01/22] Add task to scan for TV instantiations --- build-tools-internal/build.gradle | 4 + .../BaseInternalPluginBuildPlugin.java | 2 + .../LocateTransportVersionsPlugin.java | 41 ++++++ .../LocateTransportVersionsTask.java | 131 ++++++++++++++++++ server/build.gradle | 1 + 5 files changed, 179 insertions(+) create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsPlugin.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsTask.java diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index c04ba9b90d5e7..55fc01f31645f 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -220,6 +220,10 @@ gradlePlugin { id = 'elasticsearch.internal-yaml-rest-test' implementationClass = 'org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin' } + locateTransportVersionsPlugin { + id = 'elasticsearch.locate-transport-versions' + implementationClass = 'org.elasticsearch.gradle.internal.transport.LocateTransportVersionsPlugin' + } } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java index b6f4c99e3d0e6..b338985664984 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java @@ -15,6 +15,7 @@ import org.elasticsearch.gradle.internal.info.BuildParameterExtension; import org.elasticsearch.gradle.internal.precommit.JarHellPrecommitPlugin; import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin; +import org.elasticsearch.gradle.internal.transport.LocateTransportVersionsPlugin; import org.elasticsearch.gradle.plugin.PluginBuildPlugin; import org.elasticsearch.gradle.plugin.PluginPropertiesExtension; import org.elasticsearch.gradle.util.GradleUtils; @@ -36,6 +37,7 @@ public void apply(Project project) { project.getPluginManager().apply(JarHellPrecommitPlugin.class); project.getPluginManager().apply(ElasticsearchJavaPlugin.class); project.getPluginManager().apply(ClusterFeaturesMetadataPlugin.class); + project.getPluginManager().apply(LocateTransportVersionsPlugin.class); boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).getCi(); // Clear default dependencies added by public PluginBuildPlugin as we add our // own project dependencies for internal builds diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsPlugin.java new file mode 100644 index 0000000000000..624bf1f24133b --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsPlugin.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin; +import org.elasticsearch.gradle.util.GradleUtils; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.file.FileCollection; +import org.gradle.api.plugins.JavaPlugin; +import org.gradle.api.tasks.SourceSet; + +public class LocateTransportVersionsPlugin implements Plugin { + public static String TASK_NAME = "locateTransportVersions"; + public static final String TRANSPORT_VERSION_NAMES_FILE = "generated-transport-info/transport-version-set-names.txt"; + + @Override + public void apply(Project project) { + var config = project.getConfigurations().create("locateTransportVersionsConfig"); + + final var checkTransportVersion = project.getTasks().register(TASK_NAME, LocateTransportVersionsTask.class, t -> { + SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); + FileCollection dependencyJars = project.getConfigurations().getByName(JavaPlugin.COMPILE_CLASSPATH_CONFIGURATION_NAME); + FileCollection compiledPluginClasses = mainSourceSet.getOutput().getClassesDirs(); + FileCollection clasDirs = dependencyJars.plus(compiledPluginClasses) + .minus(project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME)); + t.getClassDirs().set(clasDirs); + + t.getOutputFile().set(project.getLayout().getBuildDirectory().file(TRANSPORT_VERSION_NAMES_FILE)); + }); + + project.getArtifacts().add(config.getName(), checkTransportVersion); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsTask.java new file mode 100644 index 0000000000000..14ff3be3707ae --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsTask.java @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.FileCollection; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.TaskAction; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.tree.LdcInsnNode; +import org.objectweb.asm.tree.MethodNode; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * This task locates all method invocations of org.elasticsearch.TransportVersion#getName(java.lang.String) in the + * provided directory, and then records the value of string literals passed as arguments. It then records each + * string on a newline in the provided output file. + */ +public abstract class LocateTransportVersionsTask extends DefaultTask { + public static final String TRANSPORT_VERSION_SET_CLASS = "org/elasticsearch/TransportVersion"; + public static final String TRANSPORT_VERSION_SET_METHOD_NAME = "getName"; + public static final String CLASS_EXTENSION = ".class"; + public static final String MODULE_INFO = "module-info.class"; + + /** + * The directory to scan for method invocations. + */ + @InputFiles + public abstract Property getClassDirs(); + + /** + * The output file, with each newline containing the string literal argument of each method + * invocation. + */ + @OutputFile + public abstract RegularFileProperty getOutputFile(); + + @TaskAction + public void checkTransportVersion() { + var classFiles = findJavaClassFiles(getClassDirs().get().getFiles()); + var tvNames = getTVDeclarationNames(classFiles); + + File file = getOutputFile().get().getAsFile(); + try (FileWriter writer = new FileWriter(file)) { + for (String tvName : tvNames) { + writer.write(tvName + "\n"); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static Set getTVDeclarationNames(Collection classfiles) { + var results = new HashSet(); + for (File javaFile : classfiles) { + try (InputStream inputStream = new FileInputStream(javaFile)) { + ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) { + @Override + public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { + return new MethodNode(Opcodes.ASM9, access, name, descriptor, signature, exceptions) { + @Override + public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) { + if (owner.equals(TRANSPORT_VERSION_SET_CLASS) && name.equals(TRANSPORT_VERSION_SET_METHOD_NAME)) { + var abstractInstruction = this.instructions.getLast(); + if (abstractInstruction instanceof LdcInsnNode ldcInsnNode + && ldcInsnNode.cst instanceof String tvName + && tvName.isEmpty() == false) { + results.add(tvName); + } else { + // The instruction is not a LDC with a String constant (or an empty String), + // which is not allowed. + throw new RuntimeException( + "Transport Versions must be declared with a constant and non-empty String. " + + "file: " + + javaFile.getPath() + ); + } + } + super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); + } + }; + } + }; + ClassReader classReader = new ClassReader(inputStream); + classReader.accept(classVisitor, 0); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return results; + } + + private static List findJavaClassFiles(Collection files) { + List classFiles = new ArrayList<>(); + for (File file : files) { + if (file.isDirectory()) { + File[] subFiles = file.listFiles(); + if (subFiles != null) { + classFiles.addAll(findJavaClassFiles(Arrays.asList(subFiles))); + } + } else if (file.getName().endsWith(CLASS_EXTENSION) && file.getName().endsWith(MODULE_INFO) == false) { + classFiles.add(file); + } + } + return classFiles; + } +} diff --git a/server/build.gradle b/server/build.gradle index be2b43745d0b2..6f5233be773c0 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -12,6 +12,7 @@ apply plugin: 'elasticsearch.publish' apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.internal-test-artifact' apply plugin: 'elasticsearch.test-build-info' +apply plugin: 'elasticsearch.locate-transport-versions' publishing { publications { From de27ca378e802c3765342b0b6d1f0f4f5d02f2c4 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Thu, 24 Jul 2025 11:26:28 -0700 Subject: [PATCH 02/22] Initial draft --- build-tools-internal/build.gradle | 6 +- .../BaseInternalPluginBuildPlugin.java | 4 +- ... => CollectTransportVersionNamesTask.java} | 4 +- .../LocateTransportVersionsPlugin.java | 41 ------------- ...ransportVersionGlobalManagementPlugin.java | 23 +++++++ .../TransportVersionManagementPlugin.java | 57 +++++++++++++++++ .../transport/TransportVersionUtils.java | 52 ++++++++++++++++ .../ValidateTransportVersionNamesTask.java | 61 +++++++++++++++++++ server/build.gradle | 2 +- 9 files changed, 201 insertions(+), 49 deletions(-) rename build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/{LocateTransportVersionsTask.java => CollectTransportVersionNamesTask.java} (98%) delete mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsPlugin.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionNamesTask.java diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index 55fc01f31645f..18834ceac4797 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -220,9 +220,9 @@ gradlePlugin { id = 'elasticsearch.internal-yaml-rest-test' implementationClass = 'org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin' } - locateTransportVersionsPlugin { - id = 'elasticsearch.locate-transport-versions' - implementationClass = 'org.elasticsearch.gradle.internal.transport.LocateTransportVersionsPlugin' + transportVersionNamesPlugin { + id = 'elasticsearch.transport-version-names' + implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionNamesPlugin' } } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java index b338985664984..f89eb3131bc7f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java @@ -15,7 +15,7 @@ import org.elasticsearch.gradle.internal.info.BuildParameterExtension; import org.elasticsearch.gradle.internal.precommit.JarHellPrecommitPlugin; import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin; -import org.elasticsearch.gradle.internal.transport.LocateTransportVersionsPlugin; +import org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin; import org.elasticsearch.gradle.plugin.PluginBuildPlugin; import org.elasticsearch.gradle.plugin.PluginPropertiesExtension; import org.elasticsearch.gradle.util.GradleUtils; @@ -37,7 +37,7 @@ public void apply(Project project) { project.getPluginManager().apply(JarHellPrecommitPlugin.class); project.getPluginManager().apply(ElasticsearchJavaPlugin.class); project.getPluginManager().apply(ClusterFeaturesMetadataPlugin.class); - project.getPluginManager().apply(LocateTransportVersionsPlugin.class); + project.getPluginManager().apply(TransportVersionManagementPlugin.class); boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).getCi(); // Clear default dependencies added by public PluginBuildPlugin as we add our // own project dependencies for internal builds diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java similarity index 98% rename from build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsTask.java rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java index 14ff3be3707ae..71bf8f71a4055 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java @@ -40,9 +40,9 @@ * provided directory, and then records the value of string literals passed as arguments. It then records each * string on a newline in the provided output file. */ -public abstract class LocateTransportVersionsTask extends DefaultTask { +public abstract class CollectTransportVersionNamesTask extends DefaultTask { public static final String TRANSPORT_VERSION_SET_CLASS = "org/elasticsearch/TransportVersion"; - public static final String TRANSPORT_VERSION_SET_METHOD_NAME = "getName"; + public static final String TRANSPORT_VERSION_SET_METHOD_NAME = "fromName"; public static final String CLASS_EXTENSION = ".class"; public static final String MODULE_INFO = "module-info.class"; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsPlugin.java deleted file mode 100644 index 624bf1f24133b..0000000000000 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsPlugin.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.gradle.internal.transport; - -import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin; -import org.elasticsearch.gradle.util.GradleUtils; -import org.gradle.api.Plugin; -import org.gradle.api.Project; -import org.gradle.api.file.FileCollection; -import org.gradle.api.plugins.JavaPlugin; -import org.gradle.api.tasks.SourceSet; - -public class LocateTransportVersionsPlugin implements Plugin { - public static String TASK_NAME = "locateTransportVersions"; - public static final String TRANSPORT_VERSION_NAMES_FILE = "generated-transport-info/transport-version-set-names.txt"; - - @Override - public void apply(Project project) { - var config = project.getConfigurations().create("locateTransportVersionsConfig"); - - final var checkTransportVersion = project.getTasks().register(TASK_NAME, LocateTransportVersionsTask.class, t -> { - SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); - FileCollection dependencyJars = project.getConfigurations().getByName(JavaPlugin.COMPILE_CLASSPATH_CONFIGURATION_NAME); - FileCollection compiledPluginClasses = mainSourceSet.getOutput().getClassesDirs(); - FileCollection clasDirs = dependencyJars.plus(compiledPluginClasses) - .minus(project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME)); - t.getClassDirs().set(clasDirs); - - t.getOutputFile().set(project.getLayout().getBuildDirectory().file(TRANSPORT_VERSION_NAMES_FILE)); - }); - - project.getArtifacts().add(config.getName(), checkTransportVersion); - } -} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java new file mode 100644 index 0000000000000..b64eb6dceebb8 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.gradle.api.Plugin; +import org.gradle.api.Project; + +public class TransportVersionGlobalManagementPlugin implements Plugin { + + + @Override + public void apply(Project project) { + + + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java new file mode 100644 index 0000000000000..527d0816d91f9 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.elasticsearch.gradle.util.GradleUtils; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.attributes.Attribute; +import org.gradle.api.file.FileCollection; +import org.gradle.api.tasks.SourceSet; + +public class TransportVersionManagementPlugin implements Plugin { + public static final Attribute TRANSPORT_VERSION_NAMES_ATTRIBUTE = Attribute.of("is-transport-version-names", Boolean.class); + + @Override + public void apply(Project project) { + final var transportVersionsNamesFile = "generated-transport-info/transport-version-set-names.txt"; + final var collectTask = project.getTasks().register("collectTransportVersionNames", CollectTransportVersionNamesTask.class, t -> { + t.setGroup("Transport Versions"); + t.setDescription("Collects all TransportVersion names used throughout the project"); + SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); + FileCollection clasDirs = mainSourceSet.getRuntimeClasspath(); + t.getClassDirs().set(clasDirs); + t.getOutputFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); + }); + + Configuration transportVersionsConfig = project.getConfigurations().create("transportVersions", c -> { + c.setCanBeConsumed(true); + c.setCanBeResolved(false); + c.attributes(a -> a.attribute(TRANSPORT_VERSION_NAMES_ATTRIBUTE, true)); + }); + + project.getArtifacts().add(transportVersionsConfig.getName(), collectTask); + + final var validateTask = project.getTasks().register("validateTransportVersionNames", ValidateTransportVersionNamesTask.class, t -> { + t.setGroup("Transport Versions"); + t.setDescription("Validates that all TransportVersion names used in the project have an associated data file"); + // TODO: how to ensure this is always references server? + var dir = project.getLayout().getProjectDirectory().file("src/main/resources/org/elasticsearch/transport/"); + t.getDataFileDirectory().set(dir); + t.getTransportVersionSetNamesFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); + // TODO is this correct? Needs to have both global/per-plugin versions and dependencies + t.dependsOn(collectTask); + + }); + + project.getTasks().named("check").configure(t -> t.dependsOn(validateTask)); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java new file mode 100644 index 0000000000000..2aea0c7f3eb22 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import com.google.common.collect.Comparators; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +public class TransportVersionUtils { + + public record TransportVersionSetData(String name, List ids) { + } + + public static TransportVersionSetData readDataFile(File file) { + try (BufferedReader reader = new BufferedReader(new FileReader(file, StandardCharsets.UTF_8))) { + String[] parts = reader.readLine().replaceAll("\\s+", "").split("\\|"); + if (parts.length < 2) { + throw new IllegalStateException("Invalid transport version file format, " + + "must have both name and id(s): [" + file.getAbsolutePath() + "]"); + } + if (reader.readLine() != null) { + throw new IllegalStateException("Invalid transport version file format, " + + "data file must contain only one line: [" + file.getAbsolutePath() + "]"); + } + String name = parts[0]; + var ids = Arrays.stream(parts).skip(1).map(Integer::parseInt).toList(); + var areIdsSorted = Comparators.isInOrder(ids, Comparator.naturalOrder()); + if (areIdsSorted == false) { + throw new IllegalStateException("invalid transport version file format, " + + "ids are not in order: [" + file.getAbsolutePath() + "], "); + } + return new TransportVersionSetData(name, ids); + } catch (IOException ioe) { + throw new UncheckedIOException("cannot parse transport version [" + file.getAbsolutePath() + "]", ioe); + } + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionNamesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionNamesTask.java new file mode 100644 index 0000000000000..75b6bb8e2b564 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionNamesTask.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.InputFile; +import org.gradle.api.tasks.TaskAction; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +/** + * Validates that each transport version declaration has an associated data file. + */ +public abstract class ValidateTransportVersionNamesTask extends DefaultTask { + + @InputDirectory + public abstract RegularFileProperty getDataFileDirectory(); + + @InputFile + public abstract RegularFileProperty getTransportVersionSetNamesFile(); + + @TaskAction + public void validateTransportVersions() throws IOException { + var dataFileDirectory = getDataFileDirectory(); + var tvDataDir = dataFileDirectory.getAsFile().get(); + + Set tvNamesInDataFiles = new HashSet<>(); + for (var tvDataFile : Objects.requireNonNull(tvDataDir.listFiles())) { + var data = TransportVersionUtils.readDataFile(tvDataFile); + tvNamesInDataFiles.add(data.name()); + } + + var tvSetDeclaredNamesFile = getTransportVersionSetNamesFile().get().getAsFile(); + try (var reader = new BufferedReader(new FileReader(tvSetDeclaredNamesFile))) { + reader.lines().forEach(declaredName -> { + if (tvNamesInDataFiles.contains(declaredName) == false) { + throw new RuntimeException( + "TransportVersionSetData.get(\"" + + declaredName + + "\") was used, but lacks a" + + " data file with a corresponding transport version. This can be generated with the task" // todo + ); + } + }); + } + } +} diff --git a/server/build.gradle b/server/build.gradle index 6f5233be773c0..a93a11fbe7b0c 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -12,7 +12,7 @@ apply plugin: 'elasticsearch.publish' apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.internal-test-artifact' apply plugin: 'elasticsearch.test-build-info' -apply plugin: 'elasticsearch.locate-transport-versions' +apply plugin: 'elasticsearch.transport-version-names' publishing { publications { From 68dbac8b9d22beb6f7933fbfc7788aaaa4e5a0b3 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 24 Jul 2025 14:55:21 -0700 Subject: [PATCH 03/22] wip --- build-tools-internal/build.gradle | 10 ++- .../CollectTransportVersionNamesTask.java | 2 +- ...ransportVersionGlobalManagementPlugin.java | 36 +++++++++++ .../TransportVersionManagementPlugin.java | 23 +++---- .../transport/TransportVersionUtils.java | 60 +++++++++--------- ...ValidateTransportVersionConstantsTask.java | 55 +++++++++++++++++ .../ValidateTransportVersionNamesTask.java | 61 ------------------- ...alidateTransportVersionReferencesTask.java | 52 ++++++++++++++++ server/build.gradle | 3 +- 9 files changed, 193 insertions(+), 109 deletions(-) create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java delete mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionNamesTask.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index 18834ceac4797..8c81e4ed53e09 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -220,9 +220,13 @@ gradlePlugin { id = 'elasticsearch.internal-yaml-rest-test' implementationClass = 'org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin' } - transportVersionNamesPlugin { - id = 'elasticsearch.transport-version-names' - implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionNamesPlugin' + transportVersionManagementPlugin { + id = 'elasticsearch.transport-version-management' + implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin' + } + transportVersionGlobalManagementPlugin { + id = 'elasticsearch.transport-version-global-management' + implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionGlobalManagementPlugin' } } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java index 71bf8f71a4055..ff744f111b764 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java @@ -36,7 +36,7 @@ import java.util.Set; /** - * This task locates all method invocations of org.elasticsearch.TransportVersion#getName(java.lang.String) in the + * This task locates all method invocations of org.elasticsearch.TransportVersion#fromName(java.lang.String) in the * provided directory, and then records the value of string literals passed as arguments. It then records each * string on a newline in the provided output file. */ diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java index b64eb6dceebb8..b51f46094f3be 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java @@ -11,6 +11,15 @@ import org.gradle.api.Plugin; import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.Dependency; +import org.gradle.api.artifacts.dsl.DependencyHandler; + +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TRANSPORT_VERSION_NAMES_ATTRIBUTE; +import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE; public class TransportVersionGlobalManagementPlugin implements Plugin { @@ -18,6 +27,33 @@ public class TransportVersionGlobalManagementPlugin implements Plugin { @Override public void apply(Project project) { + DependencyHandler depsHandler = project.getDependencies(); + List tvDependencies = new ArrayList<>(); + // TODO: created a named configuration so deps can be added dynamically? + for (String baseProjectPath : List.of(":modules", ":plugins", ":x-pack:plugin")) { + Project baseProject = project.project(baseProjectPath); + for (var pluginProject : baseProject.getSubprojects()) { + if (pluginProject.getParent() != baseProject) { + continue; // skip nested projects + } + tvDependencies.add(depsHandler.create(pluginProject)); + } + } + tvDependencies.add(depsHandler.create(":server")); + + Configuration tvNamesConfig = project.getConfigurations().detachedConfiguration(tvDependencies.toArray(new Dependency[0])); + tvNamesConfig.attributes(attrs -> { + attrs.attribute(ARTIFACT_TYPE_ATTRIBUTE, "csv"); + attrs.attribute(TRANSPORT_VERSION_NAMES_ATTRIBUTE, true); + }); + + var validateTask = project.getTasks().register("validateTransportVersionConstants", ValidateTransportVersionConstantsTask.class, t -> { + t.setGroup("Transport Versions"); + t.setDescription("Validates that all defined TransportVersion constants are used in at least one project"); + t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); + t.getNamesFiles().setFrom(tvNamesConfig); + }); + project.getTasks().named("check").configure(t -> t.dependsOn(validateTask)); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index 527d0816d91f9..49b0d6d8abd94 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -13,41 +13,34 @@ import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; -import org.gradle.api.attributes.Attribute; -import org.gradle.api.file.FileCollection; import org.gradle.api.tasks.SourceSet; public class TransportVersionManagementPlugin implements Plugin { - public static final Attribute TRANSPORT_VERSION_NAMES_ATTRIBUTE = Attribute.of("is-transport-version-names", Boolean.class); @Override public void apply(Project project) { - final var transportVersionsNamesFile = "generated-transport-info/transport-version-set-names.txt"; - final var collectTask = project.getTasks().register("collectTransportVersionNames", CollectTransportVersionNamesTask.class, t -> { + String transportVersionsNamesFile = "generated-transport-constants/transport-version-set-names.txt"; + var collectTask = project.getTasks().register("collectTransportVersionNames", CollectTransportVersionNamesTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Collects all TransportVersion names used throughout the project"); SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); - FileCollection clasDirs = mainSourceSet.getRuntimeClasspath(); - t.getClassDirs().set(clasDirs); + t.getClassDirs().set(mainSourceSet.getRuntimeClasspath()); t.getOutputFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); }); - Configuration transportVersionsConfig = project.getConfigurations().create("transportVersions", c -> { + Configuration transportVersionsConfig = project.getConfigurations().create("transportVersionNames", c -> { c.setCanBeConsumed(true); c.setCanBeResolved(false); - c.attributes(a -> a.attribute(TRANSPORT_VERSION_NAMES_ATTRIBUTE, true)); + c.attributes(a -> a.attribute(TransportVersionUtils.TRANSPORT_VERSION_NAMES_ATTRIBUTE, true)); }); project.getArtifacts().add(transportVersionsConfig.getName(), collectTask); - final var validateTask = project.getTasks().register("validateTransportVersionNames", ValidateTransportVersionNamesTask.class, t -> { + var validateTask = project.getTasks().register("validateTransportVersionReferences", ValidateTransportVersionReferencesTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Validates that all TransportVersion names used in the project have an associated data file"); - // TODO: how to ensure this is always references server? - var dir = project.getLayout().getProjectDirectory().file("src/main/resources/org/elasticsearch/transport/"); - t.getDataFileDirectory().set(dir); - t.getTransportVersionSetNamesFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); - // TODO is this correct? Needs to have both global/per-plugin versions and dependencies + t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); + t.getNamesFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); t.dependsOn(collectTask); }); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 2aea0c7f3eb22..29d2600e5ff36 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -11,42 +11,46 @@ import com.google.common.collect.Comparators; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; +import org.gradle.api.Project; +import org.gradle.api.attributes.Attribute; +import org.gradle.api.file.Directory; + import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; -import java.util.Arrays; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; import java.util.Comparator; import java.util.List; -public class TransportVersionUtils { +class TransportVersionUtils { - public record TransportVersionSetData(String name, List ids) { - } + static final Attribute TRANSPORT_VERSION_NAMES_ATTRIBUTE = Attribute.of("transport-version-names", Boolean.class); - public static TransportVersionSetData readDataFile(File file) { - try (BufferedReader reader = new BufferedReader(new FileReader(file, StandardCharsets.UTF_8))) { - String[] parts = reader.readLine().replaceAll("\\s+", "").split("\\|"); - if (parts.length < 2) { - throw new IllegalStateException("Invalid transport version file format, " - + "must have both name and id(s): [" + file.getAbsolutePath() + "]"); - } - if (reader.readLine() != null) { - throw new IllegalStateException("Invalid transport version file format, " - + "data file must contain only one line: [" + file.getAbsolutePath() + "]"); - } - String name = parts[0]; - var ids = Arrays.stream(parts).skip(1).map(Integer::parseInt).toList(); - var areIdsSorted = Comparators.isInOrder(ids, Comparator.naturalOrder()); - if (areIdsSorted == false) { - throw new IllegalStateException("invalid transport version file format, " - + "ids are not in order: [" + file.getAbsolutePath() + "], "); + record TransportVersionData(String name, List ids) { } + + static TransportVersionData readDataFile(Path file) throws IOException { + assert file.endsWith(".csv"); + String rawName = file.getFileName().toString(); + String name = rawName.substring(0, rawName.length() - 4); + List ids = new ArrayList<>(); + + for (String rawId : Files.readString(file, StandardCharsets.UTF_8).split(",")) { + try { + ids.add(Integer.parseInt(rawId)); + } catch (NumberFormatException e) { + throw new IOException("Failed to parse id " + rawId + " in " + file, e); } - return new TransportVersionSetData(name, ids); - } catch (IOException ioe) { - throw new UncheckedIOException("cannot parse transport version [" + file.getAbsolutePath() + "]", ioe); } + + if (Comparators.isInOrder(ids, Comparator.naturalOrder()) == false) { + throw new IOException("invalid transport version data file [" + file + "], ids are not in sorted"); + } + return new TransportVersionData(name, ids); + } + + static Directory getConstantsDirectory(Project project) { + Directory serverDir = project.getRootProject().project(":server").getLayout().getProjectDirectory(); + return serverDir.dir("src/main/resources/transport/constants"); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java new file mode 100644 index 0000000000000..b0d5aac5c6ef5 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.ConfigurableFileCollection; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.TaskAction; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; + +/** + * Validates that each defined transport version constant is referenced by at least one project. + */ +public abstract class ValidateTransportVersionConstantsTask extends DefaultTask { + + @InputDirectory + public abstract DirectoryProperty getConstantsDirectory(); + + @InputFiles + public abstract ConfigurableFileCollection getNamesFiles(); + + @TaskAction + public void validateTransportVersions() throws IOException { + Path constantsDir = getConstantsDirectory().getAsFile().get().toPath(); + + Set allTvNames = new HashSet<>(); + for (var tvNamesFile : getNamesFiles()) { + allTvNames.addAll(Files.readAllLines(tvNamesFile.toPath(), StandardCharsets.UTF_8)); + } + + try (var constantsStream = Files.list(constantsDir)) { + for (var constantsFile : constantsStream.toList()) { + var tv = TransportVersionUtils.readDataFile(constantsFile); + if (allTvNames.contains(tv.name()) == false) { + throw new IllegalStateException("Transport version constant " + tv.name() + " is not referenced"); + } + } + } + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionNamesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionNamesTask.java deleted file mode 100644 index 75b6bb8e2b564..0000000000000 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionNamesTask.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.gradle.internal.transport; - -import org.gradle.api.DefaultTask; -import org.gradle.api.file.RegularFileProperty; -import org.gradle.api.tasks.InputDirectory; -import org.gradle.api.tasks.InputFile; -import org.gradle.api.tasks.TaskAction; - -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; -import java.util.HashSet; -import java.util.Objects; -import java.util.Set; - -/** - * Validates that each transport version declaration has an associated data file. - */ -public abstract class ValidateTransportVersionNamesTask extends DefaultTask { - - @InputDirectory - public abstract RegularFileProperty getDataFileDirectory(); - - @InputFile - public abstract RegularFileProperty getTransportVersionSetNamesFile(); - - @TaskAction - public void validateTransportVersions() throws IOException { - var dataFileDirectory = getDataFileDirectory(); - var tvDataDir = dataFileDirectory.getAsFile().get(); - - Set tvNamesInDataFiles = new HashSet<>(); - for (var tvDataFile : Objects.requireNonNull(tvDataDir.listFiles())) { - var data = TransportVersionUtils.readDataFile(tvDataFile); - tvNamesInDataFiles.add(data.name()); - } - - var tvSetDeclaredNamesFile = getTransportVersionSetNamesFile().get().getAsFile(); - try (var reader = new BufferedReader(new FileReader(tvSetDeclaredNamesFile))) { - reader.lines().forEach(declaredName -> { - if (tvNamesInDataFiles.contains(declaredName) == false) { - throw new RuntimeException( - "TransportVersionSetData.get(\"" - + declaredName - + "\") was used, but lacks a" - + " data file with a corresponding transport version. This can be generated with the task" // todo - ); - } - }); - } - } -} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java new file mode 100644 index 0000000000000..4298edee309f3 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.InputFile; +import org.gradle.api.tasks.TaskAction; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Validates that each transport version named reference has a constant definition. + */ +public abstract class ValidateTransportVersionReferencesTask extends DefaultTask { + + @InputDirectory + public abstract DirectoryProperty getConstantsDirectory(); + + @InputFile + public abstract RegularFileProperty getNamesFile(); + + @TaskAction + public void validateTransportVersions() throws IOException { + Path constantsDir = getConstantsDirectory().getAsFile().get().toPath(); + Path namesFile = getNamesFile().get().getAsFile().toPath(); + + for (var tvName : Files.readAllLines(namesFile, StandardCharsets.UTF_8)) { + Path constantFile = constantsDir.resolve(tvName + ".csv"); + if (Files.exists(constantFile) == false) { + throw new RuntimeException( + "TransportVersion.fromName(\"" + + tvName + + "\") was used, but lacks a" + + " transport version constant definition. This can be generated with the task" // todo + ); + } + } + } +} diff --git a/server/build.gradle b/server/build.gradle index a93a11fbe7b0c..62ddb9738c70b 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -12,7 +12,8 @@ apply plugin: 'elasticsearch.publish' apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.internal-test-artifact' apply plugin: 'elasticsearch.test-build-info' -apply plugin: 'elasticsearch.transport-version-names' +apply plugin: 'elasticsearch.transport-version-management' +apply plugin: 'elasticsearch.transport-version-global-management' publishing { publications { From 312235e1e4945970854b3faa95d5cd7696d0ff02 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 24 Jul 2025 15:26:57 -0700 Subject: [PATCH 04/22] add example --- build-tools-internal/build.gradle | 6 +++--- ... => GlobalTransportVersionManagementPlugin.java} | 13 +++---------- .../transport/TransportVersionManagementPlugin.java | 2 +- .../internal/transport/TransportVersionUtils.java | 8 ++++++++ build.gradle | 3 ++- server/build.gradle | 1 - .../java/org/elasticsearch/TransportVersions.java | 2 +- .../transport/constants/esql_spli_on_big_values.csv | 1 + .../read/ValuesSourceReaderOperatorStatus.java | 8 ++------ 9 files changed, 21 insertions(+), 23 deletions(-) rename build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/{TransportVersionGlobalManagementPlugin.java => GlobalTransportVersionManagementPlugin.java} (83%) create mode 100644 server/src/main/resources/transport/constants/esql_spli_on_big_values.csv diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index 8c81e4ed53e09..0b81ab90ceced 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -224,9 +224,9 @@ gradlePlugin { id = 'elasticsearch.transport-version-management' implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin' } - transportVersionGlobalManagementPlugin { - id = 'elasticsearch.transport-version-global-management' - implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionGlobalManagementPlugin' + globalTransportVersionManagementPlugin { + id = 'elasticsearch.global-transport-version-management' + implementationClass = 'org.elasticsearch.gradle.internal.transport.GlobalTransportVersionManagementPlugin' } } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java similarity index 83% rename from build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index b51f46094f3be..7f2f1c48e66fd 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -18,12 +18,8 @@ import java.util.ArrayList; import java.util.List; -import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TRANSPORT_VERSION_NAMES_ATTRIBUTE; -import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE; - -public class TransportVersionGlobalManagementPlugin implements Plugin { - - +public class GlobalTransportVersionManagementPlugin implements Plugin { + @Override public void apply(Project project) { @@ -42,10 +38,7 @@ public void apply(Project project) { tvDependencies.add(depsHandler.create(":server")); Configuration tvNamesConfig = project.getConfigurations().detachedConfiguration(tvDependencies.toArray(new Dependency[0])); - tvNamesConfig.attributes(attrs -> { - attrs.attribute(ARTIFACT_TYPE_ATTRIBUTE, "csv"); - attrs.attribute(TRANSPORT_VERSION_NAMES_ATTRIBUTE, true); - }); + tvNamesConfig.attributes(TransportVersionUtils::addTransportVersionNamesAttribute); var validateTask = project.getTasks().register("validateTransportVersionConstants", ValidateTransportVersionConstantsTask.class, t -> { t.setGroup("Transport Versions"); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index 49b0d6d8abd94..3d71a790fd06e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -31,7 +31,7 @@ public void apply(Project project) { Configuration transportVersionsConfig = project.getConfigurations().create("transportVersionNames", c -> { c.setCanBeConsumed(true); c.setCanBeResolved(false); - c.attributes(a -> a.attribute(TransportVersionUtils.TRANSPORT_VERSION_NAMES_ATTRIBUTE, true)); + c.attributes(TransportVersionUtils::addTransportVersionNamesAttribute); }); project.getArtifacts().add(transportVersionsConfig.getName(), collectTask); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 29d2600e5ff36..6fa444f4f8f3e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -13,6 +13,7 @@ import org.gradle.api.Project; import org.gradle.api.attributes.Attribute; +import org.gradle.api.attributes.AttributeContainer; import org.gradle.api.file.Directory; import java.io.IOException; @@ -23,6 +24,8 @@ import java.util.Comparator; import java.util.List; +import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE; + class TransportVersionUtils { static final Attribute TRANSPORT_VERSION_NAMES_ATTRIBUTE = Attribute.of("transport-version-names", Boolean.class); @@ -53,4 +56,9 @@ static Directory getConstantsDirectory(Project project) { Directory serverDir = project.getRootProject().project(":server").getLayout().getProjectDirectory(); return serverDir.dir("src/main/resources/transport/constants"); } + + static void addTransportVersionNamesAttribute(AttributeContainer attributes) { + attributes.attribute(ARTIFACT_TYPE_ATTRIBUTE, "csv"); + attributes.attribute(TransportVersionUtils.TRANSPORT_VERSION_NAMES_ATTRIBUTE, true); + } } diff --git a/build.gradle b/build.gradle index 5b534394b4e1c..d78fa4e4919d4 100644 --- a/build.gradle +++ b/build.gradle @@ -51,6 +51,7 @@ plugins { id 'elasticsearch.repositories' id 'elasticsearch.release-tools' id 'elasticsearch.versions' + id 'elasticsearch.global-transport-version-management' id 'com.gradleup.nmcp.aggregation' } @@ -366,7 +367,7 @@ allprojects { } } } - + proj.tasks.register("bcUpgradeTest$partString") { dependsOn tasks.matching { it.name == 'bcUpgradeTest' } withReleaseBuild { diff --git a/server/build.gradle b/server/build.gradle index 62ddb9738c70b..eb4ada28e7167 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -13,7 +13,6 @@ apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.internal-test-artifact' apply plugin: 'elasticsearch.test-build-info' apply plugin: 'elasticsearch.transport-version-management' -apply plugin: 'elasticsearch.transport-version-global-management' publishing { publications { diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index dad300ae72744..d6f65733b473d 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -337,7 +337,7 @@ static TransportVersion def(int id) { public static final TransportVersion PROJECT_STATE_REGISTRY_RECORDS_DELETIONS = def(9_113_0_00); public static final TransportVersion ESQL_SERIALIZE_TIMESERIES_FIELD_TYPE = def(9_114_0_00); public static final TransportVersion ML_INFERENCE_IBM_WATSONX_COMPLETION_ADDED = def(9_115_0_00); - public static final TransportVersion ESQL_SPLIT_ON_BIG_VALUES = def(9_116_0_00); + //public static final TransportVersion ESQL_SPLIT_ON_BIG_VALUES = def(9_116_0_00); public static final TransportVersion ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS = def(9_117_0_00); public static final TransportVersion ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_TYPE = def(9_118_0_00); public static final TransportVersion ESQL_FIXED_INDEX_LIKE = def(9_119_0_00); diff --git a/server/src/main/resources/transport/constants/esql_spli_on_big_values.csv b/server/src/main/resources/transport/constants/esql_spli_on_big_values.csv new file mode 100644 index 0000000000000..3d0c94f8fa4c2 --- /dev/null +++ b/server/src/main/resources/transport/constants/esql_spli_on_big_values.csv @@ -0,0 +1 @@ +9116000,9112001,8841063 diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorStatus.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorStatus.java index 4a8fcda81f82a..bba6b9b172f0f 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorStatus.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorStatus.java @@ -23,9 +23,6 @@ import static org.elasticsearch.TransportVersions.ESQL_DOCUMENTS_FOUND_AND_VALUES_LOADED; import static org.elasticsearch.TransportVersions.ESQL_DOCUMENTS_FOUND_AND_VALUES_LOADED_8_19; -import static org.elasticsearch.TransportVersions.ESQL_SPLIT_ON_BIG_VALUES; -import static org.elasticsearch.TransportVersions.ESQL_SPLIT_ON_BIG_VALUES_8_19; -import static org.elasticsearch.TransportVersions.ESQL_SPLIT_ON_BIG_VALUES_9_1; public class ValuesSourceReaderOperatorStatus extends AbstractPageMappingToIteratorOperator.Status { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( @@ -33,6 +30,7 @@ public class ValuesSourceReaderOperatorStatus extends AbstractPageMappingToItera "values_source_reader", ValuesSourceReaderOperatorStatus::readFrom ); + static final TransportVersion SPLIT_ON_BIG_VALUES = TransportVersion.fromName("esql_split_on_big_values"); private final Map readersBuilt; private final long valuesLoaded; @@ -103,9 +101,7 @@ public void writeTo(StreamOutput out) throws IOException { } private static boolean supportsSplitOnBigValues(TransportVersion version) { - return version.onOrAfter(ESQL_SPLIT_ON_BIG_VALUES) - || version.isPatchFrom(ESQL_SPLIT_ON_BIG_VALUES_9_1) - || version.isPatchFrom(ESQL_SPLIT_ON_BIG_VALUES_8_19); + return version.supports(SPLIT_ON_BIG_VALUES); } private static boolean supportsValuesLoaded(TransportVersion version) { From 818bda4fef11549e052cab68c6c49b8a764aa266 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 24 Jul 2025 22:38:09 +0000 Subject: [PATCH 05/22] [CI] Auto commit changes from spotless --- .../GlobalTransportVersionManagementPlugin.java | 15 ++++++++------- .../TransportVersionManagementPlugin.java | 17 +++++++++-------- .../transport/TransportVersionUtils.java | 2 +- .../org/elasticsearch/TransportVersions.java | 2 +- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index 7f2f1c48e66fd..03e145c105a26 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -19,7 +19,7 @@ import java.util.List; public class GlobalTransportVersionManagementPlugin implements Plugin { - + @Override public void apply(Project project) { @@ -40,12 +40,13 @@ public void apply(Project project) { Configuration tvNamesConfig = project.getConfigurations().detachedConfiguration(tvDependencies.toArray(new Dependency[0])); tvNamesConfig.attributes(TransportVersionUtils::addTransportVersionNamesAttribute); - var validateTask = project.getTasks().register("validateTransportVersionConstants", ValidateTransportVersionConstantsTask.class, t -> { - t.setGroup("Transport Versions"); - t.setDescription("Validates that all defined TransportVersion constants are used in at least one project"); - t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); - t.getNamesFiles().setFrom(tvNamesConfig); - }); + var validateTask = project.getTasks() + .register("validateTransportVersionConstants", ValidateTransportVersionConstantsTask.class, t -> { + t.setGroup("Transport Versions"); + t.setDescription("Validates that all defined TransportVersion constants are used in at least one project"); + t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); + t.getNamesFiles().setFrom(tvNamesConfig); + }); project.getTasks().named("check").configure(t -> t.dependsOn(validateTask)); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index 3d71a790fd06e..c142b0f88ea77 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -36,14 +36,15 @@ public void apply(Project project) { project.getArtifacts().add(transportVersionsConfig.getName(), collectTask); - var validateTask = project.getTasks().register("validateTransportVersionReferences", ValidateTransportVersionReferencesTask.class, t -> { - t.setGroup("Transport Versions"); - t.setDescription("Validates that all TransportVersion names used in the project have an associated data file"); - t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); - t.getNamesFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); - t.dependsOn(collectTask); - - }); + var validateTask = project.getTasks() + .register("validateTransportVersionReferences", ValidateTransportVersionReferencesTask.class, t -> { + t.setGroup("Transport Versions"); + t.setDescription("Validates that all TransportVersion names used in the project have an associated data file"); + t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); + t.getNamesFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); + t.dependsOn(collectTask); + + }); project.getTasks().named("check").configure(t -> t.dependsOn(validateTask)); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 6fa444f4f8f3e..1fe04f2a54166 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -30,7 +30,7 @@ class TransportVersionUtils { static final Attribute TRANSPORT_VERSION_NAMES_ATTRIBUTE = Attribute.of("transport-version-names", Boolean.class); - record TransportVersionData(String name, List ids) { } + record TransportVersionData(String name, List ids) {} static TransportVersionData readDataFile(Path file) throws IOException { assert file.endsWith(".csv"); diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index d6f65733b473d..b0fe1e567cc35 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -337,7 +337,7 @@ static TransportVersion def(int id) { public static final TransportVersion PROJECT_STATE_REGISTRY_RECORDS_DELETIONS = def(9_113_0_00); public static final TransportVersion ESQL_SERIALIZE_TIMESERIES_FIELD_TYPE = def(9_114_0_00); public static final TransportVersion ML_INFERENCE_IBM_WATSONX_COMPLETION_ADDED = def(9_115_0_00); - //public static final TransportVersion ESQL_SPLIT_ON_BIG_VALUES = def(9_116_0_00); + // public static final TransportVersion ESQL_SPLIT_ON_BIG_VALUES = def(9_116_0_00); public static final TransportVersion ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS = def(9_117_0_00); public static final TransportVersion ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_TYPE = def(9_118_0_00); public static final TransportVersion ESQL_FIXED_INDEX_LIKE = def(9_119_0_00); From 1721d2e620002077cc86343c5d75e5f490573b48 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 24 Jul 2025 15:49:33 -0700 Subject: [PATCH 06/22] iter --- .../transport/GlobalTransportVersionManagementPlugin.java | 5 +++-- .../gradle/internal/transport/TransportVersionUtils.java | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index 03e145c105a26..8ccd64d841e7e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -17,6 +17,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; public class GlobalTransportVersionManagementPlugin implements Plugin { @@ -32,10 +33,10 @@ public void apply(Project project) { if (pluginProject.getParent() != baseProject) { continue; // skip nested projects } - tvDependencies.add(depsHandler.create(pluginProject)); + tvDependencies.add(depsHandler.project(Map.of("path", pluginProject.getPath()))); } } - tvDependencies.add(depsHandler.create(":server")); + tvDependencies.add(depsHandler.project(Map.of("path", ":server"))); Configuration tvNamesConfig = project.getConfigurations().detachedConfiguration(tvDependencies.toArray(new Dependency[0])); tvNamesConfig.attributes(TransportVersionUtils::addTransportVersionNamesAttribute); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 1fe04f2a54166..4dc10c4c3ca1d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -40,13 +40,13 @@ static TransportVersionData readDataFile(Path file) throws IOException { for (String rawId : Files.readString(file, StandardCharsets.UTF_8).split(",")) { try { - ids.add(Integer.parseInt(rawId)); + ids.add(Integer.parseInt(rawId.strip())); } catch (NumberFormatException e) { throw new IOException("Failed to parse id " + rawId + " in " + file, e); } } - if (Comparators.isInOrder(ids, Comparator.naturalOrder()) == false) { + if (Comparators.isInOrder(ids, Comparator.reverseOrder()) == false) { throw new IOException("invalid transport version data file [" + file + "], ids are not in sorted"); } return new TransportVersionData(name, ids); From 8c8c7c433279159cb999fc7e488b658448de2aa1 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 25 Jul 2025 09:28:10 -0700 Subject: [PATCH 07/22] iter and add generate manifest --- .../CollectTransportVersionNamesTask.java | 151 ++++++++++-------- .../GenerateTransportVersionManifestTask.java | 42 +++++ ...lobalTransportVersionManagementPlugin.java | 4 +- .../TransportVersionManagementPlugin.java | 6 +- .../transport/TransportVersionUtils.java | 35 +++- ...ValidateTransportVersionConstantsTask.java | 13 +- ...alidateTransportVersionReferencesTask.java | 15 +- ...alues.csv => esql_split_on_big_values.csv} | 0 .../ValuesSourceReaderOperatorStatus.java | 2 +- 9 files changed, 177 insertions(+), 91 deletions(-) create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java rename server/src/main/resources/transport/constants/{esql_spli_on_big_values.csv => esql_split_on_big_values.csv} (100%) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java index ff744f111b764..4aed555e41631 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java @@ -10,35 +10,36 @@ package org.elasticsearch.gradle.internal.transport; import org.gradle.api.DefaultTask; -import org.gradle.api.file.FileCollection; +import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.RegularFileProperty; -import org.gradle.api.provider.Property; +import org.gradle.api.tasks.Classpath; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.OutputFile; import org.gradle.api.tasks.TaskAction; import org.objectweb.asm.ClassReader; import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.Label; import org.objectweb.asm.MethodVisitor; import org.objectweb.asm.Opcodes; import org.objectweb.asm.tree.LdcInsnNode; import org.objectweb.asm.tree.MethodNode; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.util.HashSet; -import java.util.List; import java.util.Set; +import java.util.jar.JarInputStream; +import java.util.zip.ZipEntry; /** * This task locates all method invocations of org.elasticsearch.TransportVersion#fromName(java.lang.String) in the * provided directory, and then records the value of string literals passed as arguments. It then records each - * string on a newline in the provided output file. + * string on a newline along with path and line number in the provided output file. */ public abstract class CollectTransportVersionNamesTask extends DefaultTask { public static final String TRANSPORT_VERSION_SET_CLASS = "org/elasticsearch/TransportVersion"; @@ -50,7 +51,8 @@ public abstract class CollectTransportVersionNamesTask extends DefaultTask { * The directory to scan for method invocations. */ @InputFiles - public abstract Property getClassDirs(); + @Classpath + public abstract ConfigurableFileCollection getClassPath(); /** * The output file, with each newline containing the string literal argument of each method @@ -60,72 +62,89 @@ public abstract class CollectTransportVersionNamesTask extends DefaultTask { public abstract RegularFileProperty getOutputFile(); @TaskAction - public void checkTransportVersion() { - var classFiles = findJavaClassFiles(getClassDirs().get().getFiles()); - var tvNames = getTVDeclarationNames(classFiles); + public void checkTransportVersion() throws IOException { + var results = new HashSet(); - File file = getOutputFile().get().getAsFile(); - try (FileWriter writer = new FileWriter(file)) { - for (String tvName : tvNames) { - writer.write(tvName + "\n"); + for (var cpElement : getClassPath()) { + Path file = cpElement.toPath(); + if (Files.isDirectory(file)) { + addNamesFromClassesDirectory(results, file); + } else { + assert file.getFileName().toString().endsWith(".jar"); + addNamesFromJar(results, file); } - } catch (IOException e) { - throw new RuntimeException(e); } + + Path outputFile = getOutputFile().get().getAsFile().toPath(); + Files.writeString(outputFile, String.join("\n", results.stream().map(Object::toString).sorted().toList())); } - public static Set getTVDeclarationNames(Collection classfiles) { - var results = new HashSet(); - for (File javaFile : classfiles) { - try (InputStream inputStream = new FileInputStream(javaFile)) { - ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) { - @Override - public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { - return new MethodNode(Opcodes.ASM9, access, name, descriptor, signature, exceptions) { - @Override - public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) { - if (owner.equals(TRANSPORT_VERSION_SET_CLASS) && name.equals(TRANSPORT_VERSION_SET_METHOD_NAME)) { - var abstractInstruction = this.instructions.getLast(); - if (abstractInstruction instanceof LdcInsnNode ldcInsnNode - && ldcInsnNode.cst instanceof String tvName - && tvName.isEmpty() == false) { - results.add(tvName); - } else { - // The instruction is not a LDC with a String constant (or an empty String), - // which is not allowed. - throw new RuntimeException( - "Transport Versions must be declared with a constant and non-empty String. " - + "file: " - + javaFile.getPath() - ); - } - } - super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); - } - }; + private void addNamesFromClassesDirectory(Set results, Path file) throws IOException { + Files.walkFileTree(file, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + String filename = file.getFileName().toString(); + if (filename.endsWith(CLASS_EXTENSION) && filename.endsWith(MODULE_INFO) == false) { + try (var inputStream = Files.newInputStream(file)) { + addNamesFromClass(results, inputStream, classname(file.toString())); } - }; - ClassReader classReader = new ClassReader(inputStream); - classReader.accept(classVisitor, 0); - } catch (IOException e) { - throw new RuntimeException(e); + } + return FileVisitResult.CONTINUE; } - } - return results; + }); } - private static List findJavaClassFiles(Collection files) { - List classFiles = new ArrayList<>(); - for (File file : files) { - if (file.isDirectory()) { - File[] subFiles = file.listFiles(); - if (subFiles != null) { - classFiles.addAll(findJavaClassFiles(Arrays.asList(subFiles))); + private void addNamesFromJar(Set results, Path file) throws IOException { + try (var jar = new JarInputStream(Files.newInputStream(file))) { + ZipEntry entry; + while ((entry = jar.getNextEntry()) != null) { + String filename = entry.getName(); + if (filename.endsWith(CLASS_EXTENSION) && filename.endsWith(MODULE_INFO) == false) { + addNamesFromClass(results, jar, classname(entry.toString())); } - } else if (file.getName().endsWith(CLASS_EXTENSION) && file.getName().endsWith(MODULE_INFO) == false) { - classFiles.add(file); } } - return classFiles; + } + + private void addNamesFromClass(Set results, InputStream classBytes, String classname) + throws IOException { + ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) { + @Override + public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { + return new MethodNode(Opcodes.ASM9, access, name, descriptor, signature, exceptions) { + int lineNumber = -1; + + @Override + public void visitLineNumber(int line, Label start) { + lineNumber = line; + } + + @Override + public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) { + if (owner.equals(TRANSPORT_VERSION_SET_CLASS) && name.equals(TRANSPORT_VERSION_SET_METHOD_NAME)) { + var abstractInstruction = this.instructions.getLast(); + String location = classname + " line " + lineNumber; + if (abstractInstruction instanceof LdcInsnNode ldcInsnNode + && ldcInsnNode.cst instanceof String tvName + && tvName.isEmpty() == false) { + results.add(new TransportVersionUtils.TransportVersionReference(tvName, location)); + } else { + // The instruction is not a LDC with a String constant (or an empty String), which is not allowed. + throw new RuntimeException( + "TransportVersion.fromName must be called with a non-empty String literal. " + "See " + location + "." + ); + } + } + super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); + } + }; + } + }; + ClassReader classReader = new ClassReader(classBytes); + classReader.accept(classVisitor, 0); + } + + private static String classname(String filename) { + return filename.substring(0, filename.length() - CLASS_EXTENSION.length()).replace('/', '.'); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java new file mode 100644 index 0000000000000..374857e53ac9c --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.TaskAction; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public abstract class GenerateTransportVersionManifestTask extends DefaultTask { + @InputDirectory + public abstract DirectoryProperty getConstantsDirectory(); + + @OutputFile + public abstract RegularFileProperty getManifestFile(); + + @TaskAction + public void generateTransportVersionManifest() throws IOException { + Path constantsDir = getConstantsDirectory().get().getAsFile().toPath(); + Path manifestFile = getManifestFile().get().getAsFile().toPath(); + try (var writer = Files.newBufferedWriter(manifestFile)) { + try (var stream = Files.list(constantsDir)) { + for (String filename : stream.map(p -> p.getFileName().toString()).toList()) { + writer.write(filename + "\n"); + } + } + } + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index 8ccd64d841e7e..d6a27d20bdda7 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -39,14 +39,14 @@ public void apply(Project project) { tvDependencies.add(depsHandler.project(Map.of("path", ":server"))); Configuration tvNamesConfig = project.getConfigurations().detachedConfiguration(tvDependencies.toArray(new Dependency[0])); - tvNamesConfig.attributes(TransportVersionUtils::addTransportVersionNamesAttribute); + tvNamesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); var validateTask = project.getTasks() .register("validateTransportVersionConstants", ValidateTransportVersionConstantsTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Validates that all defined TransportVersion constants are used in at least one project"); t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); - t.getNamesFiles().setFrom(tvNamesConfig); + t.getReferencesFiles().setFrom(tvNamesConfig); }); project.getTasks().named("check").configure(t -> t.dependsOn(validateTask)); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index c142b0f88ea77..85a1259f7f7ac 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -24,14 +24,14 @@ public void apply(Project project) { t.setGroup("Transport Versions"); t.setDescription("Collects all TransportVersion names used throughout the project"); SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); - t.getClassDirs().set(mainSourceSet.getRuntimeClasspath()); + t.getClassPath().setFrom(mainSourceSet.getRuntimeClasspath()); t.getOutputFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); }); Configuration transportVersionsConfig = project.getConfigurations().create("transportVersionNames", c -> { c.setCanBeConsumed(true); c.setCanBeResolved(false); - c.attributes(TransportVersionUtils::addTransportVersionNamesAttribute); + c.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); }); project.getArtifacts().add(transportVersionsConfig.getName(), collectTask); @@ -41,7 +41,7 @@ public void apply(Project project) { t.setGroup("Transport Versions"); t.setDescription("Validates that all TransportVersion names used in the project have an associated data file"); t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); - t.getNamesFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); + t.getReferencesFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); t.dependsOn(collectTask); }); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 4dc10c4c3ca1d..e8eb8b2174280 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -28,11 +28,18 @@ class TransportVersionUtils { - static final Attribute TRANSPORT_VERSION_NAMES_ATTRIBUTE = Attribute.of("transport-version-names", Boolean.class); + static final Attribute TRANSPORT_VERSION_REFERENCES_ATTRIBUTE = Attribute.of("transport-version-references", Boolean.class); - record TransportVersionData(String name, List ids) {} + record TransportVersionConstant(String name, List ids) {} - static TransportVersionData readDataFile(Path file) throws IOException { + record TransportVersionReference(String name, String location) { + @Override + public String toString() { + return name + " " + location; + } + } + + static TransportVersionConstant readConstantFile(Path file) throws IOException { assert file.endsWith(".csv"); String rawName = file.getFileName().toString(); String name = rawName.substring(0, rawName.length() - 4); @@ -49,7 +56,20 @@ static TransportVersionData readDataFile(Path file) throws IOException { if (Comparators.isInOrder(ids, Comparator.reverseOrder()) == false) { throw new IOException("invalid transport version data file [" + file + "], ids are not in sorted"); } - return new TransportVersionData(name, ids); + return new TransportVersionConstant(name, ids); + } + + static List readReferencesFile(Path file) throws IOException { + assert file.endsWith(".txt"); + List results = new ArrayList<>(); + for (String line : Files.readAllLines(file, StandardCharsets.UTF_8)) { + String[] parts = line.split(" ", 2); + if (parts.length != 2) { + throw new IOException("Invalid transport version data file [" + file + "]: " + line); + } + results.add(new TransportVersionReference(parts[0], parts[1])); + } + return results; } static Directory getConstantsDirectory(Project project) { @@ -57,8 +77,9 @@ static Directory getConstantsDirectory(Project project) { return serverDir.dir("src/main/resources/transport/constants"); } - static void addTransportVersionNamesAttribute(AttributeContainer attributes) { - attributes.attribute(ARTIFACT_TYPE_ATTRIBUTE, "csv"); - attributes.attribute(TransportVersionUtils.TRANSPORT_VERSION_NAMES_ATTRIBUTE, true); + static void addTransportVersionReferencesAttribute(AttributeContainer attributes) { + attributes.attribute(ARTIFACT_TYPE_ATTRIBUTE, "txt"); + attributes.attribute(TransportVersionUtils.TRANSPORT_VERSION_REFERENCES_ATTRIBUTE, true); } + } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java index b0d5aac5c6ef5..c7841b777199f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java @@ -9,6 +9,7 @@ package org.elasticsearch.gradle.internal.transport; +import org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TransportVersionReference; import org.gradle.api.DefaultTask; import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.DirectoryProperty; @@ -17,12 +18,14 @@ import org.gradle.api.tasks.TaskAction; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashSet; import java.util.Set; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readConstantFile; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readReferencesFile; + /** * Validates that each defined transport version constant is referenced by at least one project. */ @@ -32,20 +35,20 @@ public abstract class ValidateTransportVersionConstantsTask extends DefaultTask public abstract DirectoryProperty getConstantsDirectory(); @InputFiles - public abstract ConfigurableFileCollection getNamesFiles(); + public abstract ConfigurableFileCollection getReferencesFiles(); @TaskAction public void validateTransportVersions() throws IOException { Path constantsDir = getConstantsDirectory().getAsFile().get().toPath(); Set allTvNames = new HashSet<>(); - for (var tvNamesFile : getNamesFiles()) { - allTvNames.addAll(Files.readAllLines(tvNamesFile.toPath(), StandardCharsets.UTF_8)); + for (var tvReferencesFile : getReferencesFiles()) { + readReferencesFile(tvReferencesFile.toPath()).stream().map(TransportVersionReference::name).forEach(allTvNames::add); } try (var constantsStream = Files.list(constantsDir)) { for (var constantsFile : constantsStream.toList()) { - var tv = TransportVersionUtils.readDataFile(constantsFile); + var tv = readConstantFile(constantsFile); if (allTvNames.contains(tv.name()) == false) { throw new IllegalStateException("Transport version constant " + tv.name() + " is not referenced"); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java index 4298edee309f3..03202065a9801 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java @@ -17,7 +17,6 @@ import org.gradle.api.tasks.TaskAction; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -30,20 +29,22 @@ public abstract class ValidateTransportVersionReferencesTask extends DefaultTask public abstract DirectoryProperty getConstantsDirectory(); @InputFile - public abstract RegularFileProperty getNamesFile(); + public abstract RegularFileProperty getReferencesFile(); @TaskAction public void validateTransportVersions() throws IOException { Path constantsDir = getConstantsDirectory().getAsFile().get().toPath(); - Path namesFile = getNamesFile().get().getAsFile().toPath(); + Path namesFile = getReferencesFile().get().getAsFile().toPath(); - for (var tvName : Files.readAllLines(namesFile, StandardCharsets.UTF_8)) { - Path constantFile = constantsDir.resolve(tvName + ".csv"); + for (var tvReference : TransportVersionUtils.readReferencesFile(namesFile)) { + Path constantFile = constantsDir.resolve(tvReference.name() + ".csv"); if (Files.exists(constantFile) == false) { throw new RuntimeException( "TransportVersion.fromName(\"" - + tvName - + "\") was used, but lacks a" + + tvReference.name() + + "\") was used at " + + tvReference.location() + + ", but lacks a" + " transport version constant definition. This can be generated with the task" // todo ); } diff --git a/server/src/main/resources/transport/constants/esql_spli_on_big_values.csv b/server/src/main/resources/transport/constants/esql_split_on_big_values.csv similarity index 100% rename from server/src/main/resources/transport/constants/esql_spli_on_big_values.csv rename to server/src/main/resources/transport/constants/esql_split_on_big_values.csv diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorStatus.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorStatus.java index bba6b9b172f0f..13ed5e4c84b8f 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorStatus.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorStatus.java @@ -30,7 +30,7 @@ public class ValuesSourceReaderOperatorStatus extends AbstractPageMappingToItera "values_source_reader", ValuesSourceReaderOperatorStatus::readFrom ); - static final TransportVersion SPLIT_ON_BIG_VALUES = TransportVersion.fromName("esql_split_on_big_values"); + private static final TransportVersion SPLIT_ON_BIG_VALUES = TransportVersion.fromName("esql_split_on_big_values"); private final Map readersBuilt; private final long valuesLoaded; From 3d3006221334402d294f8d4bb783a2e37fd3456d Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 25 Jul 2025 10:08:10 -0700 Subject: [PATCH 08/22] wired manifest task --- ...lobalTransportVersionManagementPlugin.java | 19 ++++++++++++++++--- .../TransportVersionManagementPlugin.java | 6 +++--- build.gradle | 1 - server/build.gradle | 1 + 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index d6a27d20bdda7..6618fe1610177 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -14,6 +14,8 @@ import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.Dependency; import org.gradle.api.artifacts.dsl.DependencyHandler; +import org.gradle.api.plugins.JavaPlugin; +import org.gradle.api.tasks.Copy; import java.util.ArrayList; import java.util.List; @@ -38,15 +40,26 @@ public void apply(Project project) { } tvDependencies.add(depsHandler.project(Map.of("path", ":server"))); - Configuration tvNamesConfig = project.getConfigurations().detachedConfiguration(tvDependencies.toArray(new Dependency[0])); - tvNamesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); + Configuration tvReferencesConfig = project.getConfigurations().detachedConfiguration(tvDependencies.toArray(new Dependency[0])); + tvReferencesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); + + var generateManifestTask = project.getTasks() + .register("generateTransportVersionManifest", GenerateTransportVersionManifestTask.class, t -> { + t.setGroup("Transport Versions"); + t.setDescription("Generate a manifest resource for all the known transport version constants"); + t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); + t.getManifestFile().set(project.getLayout().getBuildDirectory().file("generated-resources/manifest.txt")); + }); + project.getTasks().named(JavaPlugin.PROCESS_RESOURCES_TASK_NAME, Copy.class).configure(t -> { + t.into("transport/constants", c -> c.from(generateManifestTask)); + }); var validateTask = project.getTasks() .register("validateTransportVersionConstants", ValidateTransportVersionConstantsTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Validates that all defined TransportVersion constants are used in at least one project"); t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); - t.getReferencesFiles().setFrom(tvNamesConfig); + t.getReferencesFiles().setFrom(tvReferencesConfig); }); project.getTasks().named("check").configure(t -> t.dependsOn(validateTask)); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index 85a1259f7f7ac..af26576c15243 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -19,13 +19,13 @@ public class TransportVersionManagementPlugin implements Plugin { @Override public void apply(Project project) { - String transportVersionsNamesFile = "generated-transport-constants/transport-version-set-names.txt"; + String transportVersionReferencesFile = "generated-transport-constants/transport-version-set-names.txt"; var collectTask = project.getTasks().register("collectTransportVersionNames", CollectTransportVersionNamesTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Collects all TransportVersion names used throughout the project"); SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); t.getClassPath().setFrom(mainSourceSet.getRuntimeClasspath()); - t.getOutputFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); + t.getOutputFile().set(project.getLayout().getBuildDirectory().file(transportVersionReferencesFile)); }); Configuration transportVersionsConfig = project.getConfigurations().create("transportVersionNames", c -> { @@ -41,7 +41,7 @@ public void apply(Project project) { t.setGroup("Transport Versions"); t.setDescription("Validates that all TransportVersion names used in the project have an associated data file"); t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); - t.getReferencesFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); + t.getReferencesFile().set(project.getLayout().getBuildDirectory().file(transportVersionReferencesFile)); t.dependsOn(collectTask); }); diff --git a/build.gradle b/build.gradle index d78fa4e4919d4..f99fbc01def0e 100644 --- a/build.gradle +++ b/build.gradle @@ -51,7 +51,6 @@ plugins { id 'elasticsearch.repositories' id 'elasticsearch.release-tools' id 'elasticsearch.versions' - id 'elasticsearch.global-transport-version-management' id 'com.gradleup.nmcp.aggregation' } diff --git a/server/build.gradle b/server/build.gradle index eb4ada28e7167..59615e5f42ac5 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -13,6 +13,7 @@ apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.internal-test-artifact' apply plugin: 'elasticsearch.test-build-info' apply plugin: 'elasticsearch.transport-version-management' +apply plugin: 'elasticsearch.global-transport-version-management' publishing { publications { From d94594706e1ab85fe9f854ba72aba338060bd6a0 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 25 Jul 2025 13:30:53 -0700 Subject: [PATCH 09/22] done --- ...ollectTransportVersionReferencesTask.java} | 2 +- .../GenerateTransportVersionManifestTask.java | 4 ++-- ...lobalTransportVersionManagementPlugin.java | 22 +++++++++---------- .../TransportVersionManagementPlugin.java | 10 ++++----- .../transport/TransportVersionUtils.java | 6 ++--- ...idateTransportVersionDefinitionsTask.java} | 10 ++++----- ...alidateTransportVersionReferencesTask.java | 4 ++-- 7 files changed, 29 insertions(+), 29 deletions(-) rename build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/{CollectTransportVersionNamesTask.java => CollectTransportVersionReferencesTask.java} (98%) rename build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/{ValidateTransportVersionConstantsTask.java => ValidateTransportVersionDefinitionsTask.java} (86%) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java similarity index 98% rename from build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java index 4aed555e41631..c7e46748391aa 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java @@ -41,7 +41,7 @@ * provided directory, and then records the value of string literals passed as arguments. It then records each * string on a newline along with path and line number in the provided output file. */ -public abstract class CollectTransportVersionNamesTask extends DefaultTask { +public abstract class CollectTransportVersionReferencesTask extends DefaultTask { public static final String TRANSPORT_VERSION_SET_CLASS = "org/elasticsearch/TransportVersion"; public static final String TRANSPORT_VERSION_SET_METHOD_NAME = "fromName"; public static final String CLASS_EXTENSION = ".class"; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java index 374857e53ac9c..b568214aff77d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java @@ -22,14 +22,14 @@ public abstract class GenerateTransportVersionManifestTask extends DefaultTask { @InputDirectory - public abstract DirectoryProperty getConstantsDirectory(); + public abstract DirectoryProperty getDefinitionsDirectory(); @OutputFile public abstract RegularFileProperty getManifestFile(); @TaskAction public void generateTransportVersionManifest() throws IOException { - Path constantsDir = getConstantsDirectory().get().getAsFile().toPath(); + Path constantsDir = getDefinitionsDirectory().get().getAsFile().toPath(); Path manifestFile = getManifestFile().get().getAsFile().toPath(); try (var writer = Files.newBufferedWriter(manifestFile)) { try (var stream = Files.list(constantsDir)) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index 6618fe1610177..0afc6252e68ef 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -43,25 +43,25 @@ public void apply(Project project) { Configuration tvReferencesConfig = project.getConfigurations().detachedConfiguration(tvDependencies.toArray(new Dependency[0])); tvReferencesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); + var validateTask = project.getTasks() + .register("validateTransportVersionConstants", ValidateTransportVersionDefinitionsTask.class, t -> { + t.setGroup("Transport Versions"); + t.setDescription("Validates that all defined TransportVersion constants are used in at least one project"); + t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); + t.getReferencesFiles().setFrom(tvReferencesConfig); + }); + + project.getTasks().named("check").configure(t -> t.dependsOn(validateTask)); + var generateManifestTask = project.getTasks() .register("generateTransportVersionManifest", GenerateTransportVersionManifestTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Generate a manifest resource for all the known transport version constants"); - t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); + t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); t.getManifestFile().set(project.getLayout().getBuildDirectory().file("generated-resources/manifest.txt")); }); project.getTasks().named(JavaPlugin.PROCESS_RESOURCES_TASK_NAME, Copy.class).configure(t -> { t.into("transport/constants", c -> c.from(generateManifestTask)); }); - - var validateTask = project.getTasks() - .register("validateTransportVersionConstants", ValidateTransportVersionConstantsTask.class, t -> { - t.setGroup("Transport Versions"); - t.setDescription("Validates that all defined TransportVersion constants are used in at least one project"); - t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); - t.getReferencesFiles().setFrom(tvReferencesConfig); - }); - - project.getTasks().named("check").configure(t -> t.dependsOn(validateTask)); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index af26576c15243..03e868e372253 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -19,10 +19,10 @@ public class TransportVersionManagementPlugin implements Plugin { @Override public void apply(Project project) { - String transportVersionReferencesFile = "generated-transport-constants/transport-version-set-names.txt"; - var collectTask = project.getTasks().register("collectTransportVersionNames", CollectTransportVersionNamesTask.class, t -> { + String transportVersionReferencesFile = "transport-version/references.txt"; + var collectTask = project.getTasks().register("collectTransportVersionReferences", CollectTransportVersionReferencesTask.class, t -> { t.setGroup("Transport Versions"); - t.setDescription("Collects all TransportVersion names used throughout the project"); + t.setDescription("Collects all TransportVersion references used throughout the project"); SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); t.getClassPath().setFrom(mainSourceSet.getRuntimeClasspath()); t.getOutputFile().set(project.getLayout().getBuildDirectory().file(transportVersionReferencesFile)); @@ -39,8 +39,8 @@ public void apply(Project project) { var validateTask = project.getTasks() .register("validateTransportVersionReferences", ValidateTransportVersionReferencesTask.class, t -> { t.setGroup("Transport Versions"); - t.setDescription("Validates that all TransportVersion names used in the project have an associated data file"); - t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); + t.setDescription("Validates that all TransportVersion references used in the project have an associated definition file"); + t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); t.getReferencesFile().set(project.getLayout().getBuildDirectory().file(transportVersionReferencesFile)); t.dependsOn(collectTask); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index e8eb8b2174280..0978b38a0867b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -39,7 +39,7 @@ public String toString() { } } - static TransportVersionConstant readConstantFile(Path file) throws IOException { + static TransportVersionConstant readDefinitionFile(Path file) throws IOException { assert file.endsWith(".csv"); String rawName = file.getFileName().toString(); String name = rawName.substring(0, rawName.length() - 4); @@ -72,9 +72,9 @@ static List readReferencesFile(Path file) throws IOEx return results; } - static Directory getConstantsDirectory(Project project) { + static Directory getDefinitionsDirectory(Project project) { Directory serverDir = project.getRootProject().project(":server").getLayout().getProjectDirectory(); - return serverDir.dir("src/main/resources/transport/constants"); + return serverDir.dir("src/main/resources/transport/defined"); } static void addTransportVersionReferencesAttribute(AttributeContainer attributes) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java similarity index 86% rename from build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java index c7841b777199f..86df1344a62f0 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java @@ -23,23 +23,23 @@ import java.util.HashSet; import java.util.Set; -import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readConstantFile; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readDefinitionFile; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readReferencesFile; /** * Validates that each defined transport version constant is referenced by at least one project. */ -public abstract class ValidateTransportVersionConstantsTask extends DefaultTask { +public abstract class ValidateTransportVersionDefinitionsTask extends DefaultTask { @InputDirectory - public abstract DirectoryProperty getConstantsDirectory(); + public abstract DirectoryProperty getDefinitionsDirectory(); @InputFiles public abstract ConfigurableFileCollection getReferencesFiles(); @TaskAction public void validateTransportVersions() throws IOException { - Path constantsDir = getConstantsDirectory().getAsFile().get().toPath(); + Path constantsDir = getDefinitionsDirectory().getAsFile().get().toPath(); Set allTvNames = new HashSet<>(); for (var tvReferencesFile : getReferencesFiles()) { @@ -48,7 +48,7 @@ public void validateTransportVersions() throws IOException { try (var constantsStream = Files.list(constantsDir)) { for (var constantsFile : constantsStream.toList()) { - var tv = readConstantFile(constantsFile); + var tv = readDefinitionFile(constantsFile); if (allTvNames.contains(tv.name()) == false) { throw new IllegalStateException("Transport version constant " + tv.name() + " is not referenced"); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java index 03202065a9801..b5df69d6be012 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java @@ -26,14 +26,14 @@ public abstract class ValidateTransportVersionReferencesTask extends DefaultTask { @InputDirectory - public abstract DirectoryProperty getConstantsDirectory(); + public abstract DirectoryProperty getDefinitionsDirectory(); @InputFile public abstract RegularFileProperty getReferencesFile(); @TaskAction public void validateTransportVersions() throws IOException { - Path constantsDir = getConstantsDirectory().getAsFile().get().toPath(); + Path constantsDir = getDefinitionsDirectory().getAsFile().get().toPath(); Path namesFile = getReferencesFile().get().getAsFile().toPath(); for (var tvReference : TransportVersionUtils.readReferencesFile(namesFile)) { From c6251da0e8a4b3443dfb2b9c49d7f7fe136812f9 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 25 Jul 2025 13:32:34 -0700 Subject: [PATCH 10/22] spotless and rename --- .../TransportVersionManagementPlugin.java | 15 ++++++++------- .../esql_split_on_big_values.csv | 0 2 files changed, 8 insertions(+), 7 deletions(-) rename server/src/main/resources/transport/{constants => defined}/esql_split_on_big_values.csv (100%) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index 03e868e372253..e0a1fdb9e9aae 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -20,13 +20,14 @@ public class TransportVersionManagementPlugin implements Plugin { @Override public void apply(Project project) { String transportVersionReferencesFile = "transport-version/references.txt"; - var collectTask = project.getTasks().register("collectTransportVersionReferences", CollectTransportVersionReferencesTask.class, t -> { - t.setGroup("Transport Versions"); - t.setDescription("Collects all TransportVersion references used throughout the project"); - SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); - t.getClassPath().setFrom(mainSourceSet.getRuntimeClasspath()); - t.getOutputFile().set(project.getLayout().getBuildDirectory().file(transportVersionReferencesFile)); - }); + var collectTask = project.getTasks() + .register("collectTransportVersionReferences", CollectTransportVersionReferencesTask.class, t -> { + t.setGroup("Transport Versions"); + t.setDescription("Collects all TransportVersion references used throughout the project"); + SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); + t.getClassPath().setFrom(mainSourceSet.getRuntimeClasspath()); + t.getOutputFile().set(project.getLayout().getBuildDirectory().file(transportVersionReferencesFile)); + }); Configuration transportVersionsConfig = project.getConfigurations().create("transportVersionNames", c -> { c.setCanBeConsumed(true); diff --git a/server/src/main/resources/transport/constants/esql_split_on_big_values.csv b/server/src/main/resources/transport/defined/esql_split_on_big_values.csv similarity index 100% rename from server/src/main/resources/transport/constants/esql_split_on_big_values.csv rename to server/src/main/resources/transport/defined/esql_split_on_big_values.csv From 97503ab5aedc07e1c9f3289a0ba0013357d282e8 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 25 Jul 2025 13:41:57 -0700 Subject: [PATCH 11/22] moved to defined --- server/src/main/java/org/elasticsearch/TransportVersion.java | 4 ++-- .../resources/transport/{constant => defined}/manifest.txt | 0 .../test/resources/transport/{constant => defined}/test_0.csv | 0 .../test/resources/transport/{constant => defined}/test_1.csv | 0 .../test/resources/transport/{constant => defined}/test_2.csv | 0 .../test/resources/transport/{constant => defined}/test_3.csv | 0 .../test/resources/transport/{constant => defined}/test_4.csv | 0 7 files changed, 2 insertions(+), 2 deletions(-) rename server/src/test/resources/transport/{constant => defined}/manifest.txt (100%) rename server/src/test/resources/transport/{constant => defined}/test_0.csv (100%) rename server/src/test/resources/transport/{constant => defined}/test_1.csv (100%) rename server/src/test/resources/transport/{constant => defined}/test_2.csv (100%) rename server/src/test/resources/transport/{constant => defined}/test_3.csv (100%) rename server/src/test/resources/transport/{constant => defined}/test_4.csv (100%) diff --git a/server/src/main/java/org/elasticsearch/TransportVersion.java b/server/src/main/java/org/elasticsearch/TransportVersion.java index 2ac4c1bf72ab6..d31bb2a5e5495 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersion.java +++ b/server/src/main/java/org/elasticsearch/TransportVersion.java @@ -399,7 +399,7 @@ private static Map loadTransportVersionsByName() { throw new UncheckedIOException("latest transport version file not found at [" + latestLocation + "]", ioe); } - String manifestLocation = "/transport/constant/manifest.txt"; + String manifestLocation = "/transport/defined/manifest.txt"; List versionFileNames = null; if (latestId > -1) { try (InputStream inputStream = TransportVersion.class.getResourceAsStream(manifestLocation)) { @@ -414,7 +414,7 @@ private static Map loadTransportVersionsByName() { if (versionFileNames != null) { for (String name : versionFileNames) { - String versionLocation = "/transport/constant/" + name; + String versionLocation = "/transport/defined/" + name; try (InputStream inputStream = TransportVersion.class.getResourceAsStream(versionLocation)) { if (inputStream == null) { throw new IllegalStateException("transport version file not found at [" + versionLocation + "]"); diff --git a/server/src/test/resources/transport/constant/manifest.txt b/server/src/test/resources/transport/defined/manifest.txt similarity index 100% rename from server/src/test/resources/transport/constant/manifest.txt rename to server/src/test/resources/transport/defined/manifest.txt diff --git a/server/src/test/resources/transport/constant/test_0.csv b/server/src/test/resources/transport/defined/test_0.csv similarity index 100% rename from server/src/test/resources/transport/constant/test_0.csv rename to server/src/test/resources/transport/defined/test_0.csv diff --git a/server/src/test/resources/transport/constant/test_1.csv b/server/src/test/resources/transport/defined/test_1.csv similarity index 100% rename from server/src/test/resources/transport/constant/test_1.csv rename to server/src/test/resources/transport/defined/test_1.csv diff --git a/server/src/test/resources/transport/constant/test_2.csv b/server/src/test/resources/transport/defined/test_2.csv similarity index 100% rename from server/src/test/resources/transport/constant/test_2.csv rename to server/src/test/resources/transport/defined/test_2.csv diff --git a/server/src/test/resources/transport/constant/test_3.csv b/server/src/test/resources/transport/defined/test_3.csv similarity index 100% rename from server/src/test/resources/transport/constant/test_3.csv rename to server/src/test/resources/transport/defined/test_3.csv diff --git a/server/src/test/resources/transport/constant/test_4.csv b/server/src/test/resources/transport/defined/test_4.csv similarity index 100% rename from server/src/test/resources/transport/constant/test_4.csv rename to server/src/test/resources/transport/defined/test_4.csv From 508aec1c5a1014472d2b73de3f208f0ec8f4a54e Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 25 Jul 2025 13:47:47 -0700 Subject: [PATCH 12/22] fix task name --- .../transport/GlobalTransportVersionManagementPlugin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index 0afc6252e68ef..bb92ac9a1a226 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -44,7 +44,7 @@ public void apply(Project project) { tvReferencesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); var validateTask = project.getTasks() - .register("validateTransportVersionConstants", ValidateTransportVersionDefinitionsTask.class, t -> { + .register("validateTransportVersionDefinitions", ValidateTransportVersionDefinitionsTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Validates that all defined TransportVersion constants are used in at least one project"); t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); From 1e3fb6503c94a6a3f77f5237e5d2108ffcfdec01 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 25 Jul 2025 15:39:59 -0700 Subject: [PATCH 13/22] address review comments --- .gitignore | 1 + .../CollectTransportVersionReferencesTask.java | 5 +++-- .../GlobalTransportVersionManagementPlugin.java | 4 +++- .../transport/TransportVersionManagementPlugin.java | 12 ++++++------ .../internal/transport/TransportVersionUtils.java | 2 +- .../ValidateTransportVersionDefinitionsTask.java | 2 ++ .../ValidateTransportVersionReferencesTask.java | 2 ++ 7 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 8b2da4dc0832a..cac5a799012e1 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,7 @@ testfixtures_shared/ # Generated checkstyle_ide.xml x-pack/plugin/esql/src/main/generated-src/generated/ +server/src/main/resources/transport/defined/manifest.txt # JEnv .java-version diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java index c7e46748391aa..c3f3470413b7f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java @@ -12,6 +12,7 @@ import org.gradle.api.DefaultTask; import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.Classpath; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.OutputFile; @@ -41,6 +42,7 @@ * provided directory, and then records the value of string literals passed as arguments. It then records each * string on a newline along with path and line number in the provided output file. */ +@CacheableTask public abstract class CollectTransportVersionReferencesTask extends DefaultTask { public static final String TRANSPORT_VERSION_SET_CLASS = "org/elasticsearch/TransportVersion"; public static final String TRANSPORT_VERSION_SET_METHOD_NAME = "fromName"; @@ -50,7 +52,6 @@ public abstract class CollectTransportVersionReferencesTask extends DefaultTask /** * The directory to scan for method invocations. */ - @InputFiles @Classpath public abstract ConfigurableFileCollection getClassPath(); @@ -145,6 +146,6 @@ public void visitMethodInsn(int opcode, String owner, String name, String descri } private static String classname(String filename) { - return filename.substring(0, filename.length() - CLASS_EXTENSION.length()).replace('/', '.'); + return filename.substring(0, filename.length() - CLASS_EXTENSION.length()).replaceAll("[/\\\\]", "."); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index bb92ac9a1a226..323706de7dfa6 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -16,6 +16,7 @@ import org.gradle.api.artifacts.dsl.DependencyHandler; import org.gradle.api.plugins.JavaPlugin; import org.gradle.api.tasks.Copy; +import org.gradle.language.base.plugins.LifecycleBasePlugin; import java.util.ArrayList; import java.util.List; @@ -25,6 +26,7 @@ public class GlobalTransportVersionManagementPlugin implements Plugin { @Override public void apply(Project project) { + project.getPluginManager().apply(LifecycleBasePlugin.class); DependencyHandler depsHandler = project.getDependencies(); List tvDependencies = new ArrayList<>(); @@ -51,7 +53,7 @@ public void apply(Project project) { t.getReferencesFiles().setFrom(tvReferencesConfig); }); - project.getTasks().named("check").configure(t -> t.dependsOn(validateTask)); + project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(validateTask)); var generateManifestTask = project.getTasks() .register("generateTransportVersionManifest", GenerateTransportVersionManifestTask.class, t -> { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index e0a1fdb9e9aae..04ee595e67e64 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -14,19 +14,21 @@ import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; import org.gradle.api.tasks.SourceSet; +import org.gradle.language.base.plugins.LifecycleBasePlugin; public class TransportVersionManagementPlugin implements Plugin { @Override public void apply(Project project) { - String transportVersionReferencesFile = "transport-version/references.txt"; + project.getPluginManager().apply(LifecycleBasePlugin.class); + var collectTask = project.getTasks() .register("collectTransportVersionReferences", CollectTransportVersionReferencesTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Collects all TransportVersion references used throughout the project"); SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); t.getClassPath().setFrom(mainSourceSet.getRuntimeClasspath()); - t.getOutputFile().set(project.getLayout().getBuildDirectory().file(transportVersionReferencesFile)); + t.getOutputFile().set(project.getLayout().getBuildDirectory().file("transport-version/references.txt")); }); Configuration transportVersionsConfig = project.getConfigurations().create("transportVersionNames", c -> { @@ -42,11 +44,9 @@ public void apply(Project project) { t.setGroup("Transport Versions"); t.setDescription("Validates that all TransportVersion references used in the project have an associated definition file"); t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); - t.getReferencesFile().set(project.getLayout().getBuildDirectory().file(transportVersionReferencesFile)); - t.dependsOn(collectTask); - + t.getReferencesFile().set(collectTask.get().getOutputFile()); }); - project.getTasks().named("check").configure(t -> t.dependsOn(validateTask)); + project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(validateTask)); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 0978b38a0867b..68716a5d6399c 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -73,7 +73,7 @@ static List readReferencesFile(Path file) throws IOEx } static Directory getDefinitionsDirectory(Project project) { - Directory serverDir = project.getRootProject().project(":server").getLayout().getProjectDirectory(); + Directory serverDir = project.project(":server").getLayout().getProjectDirectory(); return serverDir.dir("src/main/resources/transport/defined"); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java index 86df1344a62f0..f5d079dddb204 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java @@ -13,6 +13,7 @@ import org.gradle.api.DefaultTask; import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.InputDirectory; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.TaskAction; @@ -29,6 +30,7 @@ /** * Validates that each defined transport version constant is referenced by at least one project. */ +@CacheableTask public abstract class ValidateTransportVersionDefinitionsTask extends DefaultTask { @InputDirectory diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java index b5df69d6be012..bafd7bc2e594f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java @@ -12,6 +12,7 @@ import org.gradle.api.DefaultTask; import org.gradle.api.file.DirectoryProperty; import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.InputDirectory; import org.gradle.api.tasks.InputFile; import org.gradle.api.tasks.TaskAction; @@ -23,6 +24,7 @@ /** * Validates that each transport version named reference has a constant definition. */ +@CacheableTask public abstract class ValidateTransportVersionReferencesTask extends DefaultTask { @InputDirectory From 49dca335ef3c1e116432df63b46d731cd40dd710 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 25 Jul 2025 15:50:21 -0700 Subject: [PATCH 14/22] iter --- ...lobalTransportVersionManagementPlugin.java | 25 ++++++++----------- .../TransportVersionManagementPlugin.java | 11 +++----- .../transport/TransportVersionUtils.java | 8 ++++-- 3 files changed, 19 insertions(+), 25 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index 323706de7dfa6..bf74f52ee1912 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -9,6 +9,7 @@ package org.elasticsearch.gradle.internal.transport; +import org.elasticsearch.gradle.internal.BaseInternalPluginBuildPlugin; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; @@ -29,22 +30,17 @@ public void apply(Project project) { project.getPluginManager().apply(LifecycleBasePlugin.class); DependencyHandler depsHandler = project.getDependencies(); - List tvDependencies = new ArrayList<>(); - // TODO: created a named configuration so deps can be added dynamically? - for (String baseProjectPath : List.of(":modules", ":plugins", ":x-pack:plugin")) { - Project baseProject = project.project(baseProjectPath); - for (var pluginProject : baseProject.getSubprojects()) { - if (pluginProject.getParent() != baseProject) { - continue; // skip nested projects - } - tvDependencies.add(depsHandler.project(Map.of("path", pluginProject.getPath()))); - } - } - tvDependencies.add(depsHandler.project(Map.of("path", ":server"))); - - Configuration tvReferencesConfig = project.getConfigurations().detachedConfiguration(tvDependencies.toArray(new Dependency[0])); + Dependency selfDependency = depsHandler.project(Map.of("path", project.getPath())); + Configuration tvReferencesConfig = project.getConfigurations().detachedConfiguration(selfDependency); tvReferencesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); + // iterate through all projects, and if the ES plugin build plugin is applied, add that project back as a dep to check + for (Project subProject : project.getRootProject().getSubprojects()) { + subProject.getPlugins().withType(BaseInternalPluginBuildPlugin.class).configureEach(plugin -> { + tvReferencesConfig.getDependencies().add(depsHandler.project(Map.of("path", subProject.getPath()))); + }); + } + var validateTask = project.getTasks() .register("validateTransportVersionDefinitions", ValidateTransportVersionDefinitionsTask.class, t -> { t.setGroup("Transport Versions"); @@ -52,7 +48,6 @@ public void apply(Project project) { t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); t.getReferencesFiles().setFrom(tvReferencesConfig); }); - project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(validateTask)); var generateManifestTask = project.getTasks() diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index 04ee595e67e64..6b11756cc8352 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -31,13 +31,9 @@ public void apply(Project project) { t.getOutputFile().set(project.getLayout().getBuildDirectory().file("transport-version/references.txt")); }); - Configuration transportVersionsConfig = project.getConfigurations().create("transportVersionNames", c -> { - c.setCanBeConsumed(true); - c.setCanBeResolved(false); - c.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); - }); - - project.getArtifacts().add(transportVersionsConfig.getName(), collectTask); + Configuration tvReferencesConfig = project.getConfigurations().detachedConfiguration(); + tvReferencesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); + project.getArtifacts().add(tvReferencesConfig.getName(), collectTask); var validateTask = project.getTasks() .register("validateTransportVersionReferences", ValidateTransportVersionReferencesTask.class, t -> { @@ -46,7 +42,6 @@ public void apply(Project project) { t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); t.getReferencesFile().set(collectTask.get().getOutputFile()); }); - project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(validateTask)); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 68716a5d6399c..ca932adadae25 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -73,8 +73,12 @@ static List readReferencesFile(Path file) throws IOEx } static Directory getDefinitionsDirectory(Project project) { - Directory serverDir = project.project(":server").getLayout().getProjectDirectory(); - return serverDir.dir("src/main/resources/transport/defined"); + var projectName = project.findProperty("org.elasticsearch.transport.definitionsProject"); + if (projectName == null) { + projectName = ":server"; + } + Directory projectDir = project.project(projectName.toString()).getLayout().getProjectDirectory(); + return projectDir.dir("src/main/resources/transport/defined"); } static void addTransportVersionReferencesAttribute(AttributeContainer attributes) { From a4c62df246f2b19b7ae1495659a84c8be28207f0 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 25 Jul 2025 18:40:59 -0700 Subject: [PATCH 15/22] more feedback --- .../CollectTransportVersionReferencesTask.java | 18 ++---------------- ...GlobalTransportVersionManagementPlugin.java | 7 +++---- .../TransportVersionManagementPlugin.java | 9 ++++++--- ...alidateTransportVersionDefinitionsTask.java | 4 ++++ ...ValidateTransportVersionReferencesTask.java | 4 ++++ x-pack/plugin/esql/compute/build.gradle | 1 + 6 files changed, 20 insertions(+), 23 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java index c3f3470413b7f..e4de8a0051366 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java @@ -14,8 +14,9 @@ import org.gradle.api.file.RegularFileProperty; import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.Classpath; -import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskAction; import org.objectweb.asm.ClassReader; import org.objectweb.asm.ClassVisitor; @@ -70,9 +71,6 @@ public void checkTransportVersion() throws IOException { Path file = cpElement.toPath(); if (Files.isDirectory(file)) { addNamesFromClassesDirectory(results, file); - } else { - assert file.getFileName().toString().endsWith(".jar"); - addNamesFromJar(results, file); } } @@ -95,18 +93,6 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO }); } - private void addNamesFromJar(Set results, Path file) throws IOException { - try (var jar = new JarInputStream(Files.newInputStream(file))) { - ZipEntry entry; - while ((entry = jar.getNextEntry()) != null) { - String filename = entry.getName(); - if (filename.endsWith(CLASS_EXTENSION) && filename.endsWith(MODULE_INFO) == false) { - addNamesFromClass(results, jar, classname(entry.toString())); - } - } - } - } - private void addNamesFromClass(Set results, InputStream classBytes, String classname) throws IOException { ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index bf74f52ee1912..3d96c3868d885 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -30,13 +30,12 @@ public void apply(Project project) { project.getPluginManager().apply(LifecycleBasePlugin.class); DependencyHandler depsHandler = project.getDependencies(); - Dependency selfDependency = depsHandler.project(Map.of("path", project.getPath())); - Configuration tvReferencesConfig = project.getConfigurations().detachedConfiguration(selfDependency); + Configuration tvReferencesConfig = project.getConfigurations().detachedConfiguration(); tvReferencesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); - // iterate through all projects, and if the ES plugin build plugin is applied, add that project back as a dep to check + // iterate through all projects, and if the management plugin is applied, add that project back as a dep to check for (Project subProject : project.getRootProject().getSubprojects()) { - subProject.getPlugins().withType(BaseInternalPluginBuildPlugin.class).configureEach(plugin -> { + subProject.getPlugins().withType(TransportVersionManagementPlugin.class).configureEach(plugin -> { tvReferencesConfig.getDependencies().add(depsHandler.project(Map.of("path", subProject.getPath()))); }); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index 6b11756cc8352..b168aed47f5f1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -27,12 +27,15 @@ public void apply(Project project) { t.setGroup("Transport Versions"); t.setDescription("Collects all TransportVersion references used throughout the project"); SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); - t.getClassPath().setFrom(mainSourceSet.getRuntimeClasspath()); + t.getClassPath().setFrom(mainSourceSet.getOutput()); t.getOutputFile().set(project.getLayout().getBuildDirectory().file("transport-version/references.txt")); }); - Configuration tvReferencesConfig = project.getConfigurations().detachedConfiguration(); - tvReferencesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); + Configuration tvReferencesConfig = project.getConfigurations().create("transportVersionReferences", c -> { + c.setCanBeConsumed(true); + c.setCanBeResolved(false); + c.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); + }); project.getArtifacts().add(tvReferencesConfig.getName(), collectTask); var validateTask = project.getTasks() diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java index f5d079dddb204..65941f1a245bc 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java @@ -16,6 +16,8 @@ import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.InputDirectory; import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskAction; import java.io.IOException; @@ -34,9 +36,11 @@ public abstract class ValidateTransportVersionDefinitionsTask extends DefaultTask { @InputDirectory + @PathSensitive(PathSensitivity.RELATIVE) public abstract DirectoryProperty getDefinitionsDirectory(); @InputFiles + @PathSensitive(PathSensitivity.RELATIVE) public abstract ConfigurableFileCollection getReferencesFiles(); @TaskAction diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java index bafd7bc2e594f..f5b8126977db4 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java @@ -15,6 +15,8 @@ import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.InputDirectory; import org.gradle.api.tasks.InputFile; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskAction; import java.io.IOException; @@ -28,9 +30,11 @@ public abstract class ValidateTransportVersionReferencesTask extends DefaultTask { @InputDirectory + @PathSensitive(PathSensitivity.RELATIVE) public abstract DirectoryProperty getDefinitionsDirectory(); @InputFile + @PathSensitive(PathSensitivity.RELATIVE) public abstract RegularFileProperty getReferencesFile(); @TaskAction diff --git a/x-pack/plugin/esql/compute/build.gradle b/x-pack/plugin/esql/compute/build.gradle index 66867ae668fcc..ebdf3074e9c8b 100644 --- a/x-pack/plugin/esql/compute/build.gradle +++ b/x-pack/plugin/esql/compute/build.gradle @@ -3,6 +3,7 @@ import org.elasticsearch.gradle.internal.util.SourceDirectoryCommandLineArgument apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.string-templates' apply plugin: 'elasticsearch.publish' +apply plugin: 'elasticsearch.transport-version-management' base { archivesName = 'x-pack-esql-compute' From 5aab19cb26596aa8fb0f6d05392b4d0fec480f49 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Sat, 26 Jul 2025 01:55:24 +0000 Subject: [PATCH 16/22] [CI] Auto commit changes from spotless --- .../transport/CollectTransportVersionReferencesTask.java | 4 ---- .../transport/GlobalTransportVersionManagementPlugin.java | 4 ---- 2 files changed, 8 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java index e4de8a0051366..76d0d48f0db1f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java @@ -15,8 +15,6 @@ import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.Classpath; import org.gradle.api.tasks.OutputFile; -import org.gradle.api.tasks.PathSensitive; -import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskAction; import org.objectweb.asm.ClassReader; import org.objectweb.asm.ClassVisitor; @@ -35,8 +33,6 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.HashSet; import java.util.Set; -import java.util.jar.JarInputStream; -import java.util.zip.ZipEntry; /** * This task locates all method invocations of org.elasticsearch.TransportVersion#fromName(java.lang.String) in the diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index 3d96c3868d885..9b2f666da46a1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -9,18 +9,14 @@ package org.elasticsearch.gradle.internal.transport; -import org.elasticsearch.gradle.internal.BaseInternalPluginBuildPlugin; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; -import org.gradle.api.artifacts.Dependency; import org.gradle.api.artifacts.dsl.DependencyHandler; import org.gradle.api.plugins.JavaPlugin; import org.gradle.api.tasks.Copy; import org.gradle.language.base.plugins.LifecycleBasePlugin; -import java.util.ArrayList; -import java.util.List; import java.util.Map; public class GlobalTransportVersionManagementPlugin implements Plugin { From 1cfa4121824cc7441a71cbe6a86d5e340a780602 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 29 Jul 2025 08:34:18 -0700 Subject: [PATCH 17/22] wip --- .../transport/GlobalTransportVersionManagementPlugin.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index 9b2f666da46a1..622c1aa2a349c 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -26,7 +26,9 @@ public void apply(Project project) { project.getPluginManager().apply(LifecycleBasePlugin.class); DependencyHandler depsHandler = project.getDependencies(); - Configuration tvReferencesConfig = project.getConfigurations().detachedConfiguration(); + Configuration tvReferencesConfig = project.getConfigurations().create("globalTvReferences"); + tvReferencesConfig.setCanBeConsumed(false); + tvReferencesConfig.setCanBeResolved(true); tvReferencesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); // iterate through all projects, and if the management plugin is applied, add that project back as a dep to check @@ -48,12 +50,12 @@ public void apply(Project project) { var generateManifestTask = project.getTasks() .register("generateTransportVersionManifest", GenerateTransportVersionManifestTask.class, t -> { t.setGroup("Transport Versions"); - t.setDescription("Generate a manifest resource for all the known transport version constants"); + t.setDescription("Generate a manifest resource for all the known transport version definitions"); t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); t.getManifestFile().set(project.getLayout().getBuildDirectory().file("generated-resources/manifest.txt")); }); project.getTasks().named(JavaPlugin.PROCESS_RESOURCES_TASK_NAME, Copy.class).configure(t -> { - t.into("transport/constants", c -> c.from(generateManifestTask)); + t.into("transport/defined", c -> c.from(generateManifestTask)); }); } } From 3dd41ffcb1a7db749e30b1da5ffb5ab7aaa6fb46 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 30 Jul 2025 12:42:23 -0700 Subject: [PATCH 18/22] remove constants --- server/src/main/java/org/elasticsearch/TransportVersions.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 182c6a766dfb4..397ccefd8b955 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -211,7 +211,6 @@ static TransportVersion def(int id) { public static final TransportVersion ML_INFERENCE_COHERE_API_VERSION_8_19 = def(8_841_0_60); public static final TransportVersion ESQL_DOCUMENTS_FOUND_AND_VALUES_LOADED_8_19 = def(8_841_0_61); public static final TransportVersion ESQL_PROFILE_INCLUDE_PLAN_8_19 = def(8_841_0_62); - public static final TransportVersion ESQL_SPLIT_ON_BIG_VALUES_8_19 = def(8_841_0_63); public static final TransportVersion ESQL_FIXED_INDEX_LIKE_8_19 = def(8_841_0_64); public static final TransportVersion V_9_0_0 = def(9_000_0_09); public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_1 = def(9_000_0_10); @@ -330,7 +329,6 @@ static TransportVersion def(int id) { public static final TransportVersion ML_INFERENCE_COHERE_API_VERSION = def(9_110_0_00); public static final TransportVersion ESQL_PROFILE_INCLUDE_PLAN = def(9_111_0_00); public static final TransportVersion MAPPINGS_IN_DATA_STREAMS = def(9_112_0_00); - public static final TransportVersion ESQL_SPLIT_ON_BIG_VALUES_9_1 = def(9_112_0_01); public static final TransportVersion ESQL_FIXED_INDEX_LIKE_9_1 = def(9_112_0_02); public static final TransportVersion ESQL_SAMPLE_OPERATOR_STATUS_9_1 = def(9_112_0_03); // Below is the first version in 9.2 and NOT in 9.1. @@ -338,7 +336,6 @@ static TransportVersion def(int id) { public static final TransportVersion PROJECT_STATE_REGISTRY_RECORDS_DELETIONS = def(9_113_0_00); public static final TransportVersion ESQL_SERIALIZE_TIMESERIES_FIELD_TYPE = def(9_114_0_00); public static final TransportVersion ML_INFERENCE_IBM_WATSONX_COMPLETION_ADDED = def(9_115_0_00); - // public static final TransportVersion ESQL_SPLIT_ON_BIG_VALUES = def(9_116_0_00); public static final TransportVersion ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS = def(9_117_0_00); public static final TransportVersion ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_TYPE = def(9_118_0_00); public static final TransportVersion ESQL_FIXED_INDEX_LIKE = def(9_119_0_00); From 3c465d4e3f856510ede2c78b5cd1888812c39bd3 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 31 Jul 2025 09:00:02 -0700 Subject: [PATCH 19/22] make definitions dir optional for validation --- .../transport/GenerateTransportVersionManifestTask.java | 4 ++++ .../transport/ValidateTransportVersionReferencesTask.java | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java index b568214aff77d..47ab03e80c31c 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java @@ -34,6 +34,10 @@ public void generateTransportVersionManifest() throws IOException { try (var writer = Files.newBufferedWriter(manifestFile)) { try (var stream = Files.list(constantsDir)) { for (String filename : stream.map(p -> p.getFileName().toString()).toList()) { + if (filename.equals(manifestFile.getFileName().toString())) { + // don't list self + continue; + } writer.write(filename + "\n"); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java index f5b8126977db4..01f90e4ef1f85 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java @@ -15,6 +15,7 @@ import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.InputDirectory; import org.gradle.api.tasks.InputFile; +import org.gradle.api.tasks.Optional; import org.gradle.api.tasks.PathSensitive; import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskAction; @@ -30,6 +31,7 @@ public abstract class ValidateTransportVersionReferencesTask extends DefaultTask { @InputDirectory + @Optional @PathSensitive(PathSensitivity.RELATIVE) public abstract DirectoryProperty getDefinitionsDirectory(); @@ -39,11 +41,11 @@ public abstract class ValidateTransportVersionReferencesTask extends DefaultTask @TaskAction public void validateTransportVersions() throws IOException { - Path constantsDir = getDefinitionsDirectory().getAsFile().get().toPath(); + Path definitionsDir = getDefinitionsDirectory().getAsFile().get().toPath(); Path namesFile = getReferencesFile().get().getAsFile().toPath(); for (var tvReference : TransportVersionUtils.readReferencesFile(namesFile)) { - Path constantFile = constantsDir.resolve(tvReference.name() + ".csv"); + Path constantFile = definitionsDir.resolve(tvReference.name() + ".csv"); if (Files.exists(constantFile) == false) { throw new RuntimeException( "TransportVersion.fromName(\"" From 6821e7076a8c47b81c2dd67211eaed3f73018b45 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 31 Jul 2025 10:02:23 -0700 Subject: [PATCH 20/22] better reference checker --- .../ValidateTransportVersionReferencesTask.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java index 01f90e4ef1f85..79050f1b171a5 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.function.Predicate; /** * Validates that each transport version named reference has a constant definition. @@ -41,19 +42,24 @@ public abstract class ValidateTransportVersionReferencesTask extends DefaultTask @TaskAction public void validateTransportVersions() throws IOException { - Path definitionsDir = getDefinitionsDirectory().getAsFile().get().toPath(); + final Predicate referenceChecker; + if (getDefinitionsDirectory().isPresent()) { + Path definitionsDir = getDefinitionsDirectory().getAsFile().get().toPath(); + referenceChecker = (name) -> Files.exists(definitionsDir.resolve(name + ".csv")); + } else { + referenceChecker = (name) -> false; + } Path namesFile = getReferencesFile().get().getAsFile().toPath(); for (var tvReference : TransportVersionUtils.readReferencesFile(namesFile)) { - Path constantFile = definitionsDir.resolve(tvReference.name() + ".csv"); - if (Files.exists(constantFile) == false) { + if (referenceChecker.test(tvReference.name()) == false) { throw new RuntimeException( "TransportVersion.fromName(\"" + tvReference.name() + "\") was used at " + tvReference.location() + ", but lacks a" - + " transport version constant definition. This can be generated with the task" // todo + + " transport version definition. This can be generated with the task" // todo ); } } From aa149650b38ff94d20c170d5e9d514d47fb477c0 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 1 Aug 2025 09:35:31 -0700 Subject: [PATCH 21/22] try again --- .../transport/TransportVersionManagementPlugin.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index b168aed47f5f1..47af8f288f958 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -13,6 +13,7 @@ import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; +import org.gradle.api.file.Directory; import org.gradle.api.tasks.SourceSet; import org.gradle.language.base.plugins.LifecycleBasePlugin; @@ -42,7 +43,10 @@ public void apply(Project project) { .register("validateTransportVersionReferences", ValidateTransportVersionReferencesTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Validates that all TransportVersion references used in the project have an associated definition file"); - t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); + Directory definitionsDir = TransportVersionUtils.getDefinitionsDirectory(project); + if (definitionsDir.getAsFile().exists()) { + t.getDefinitionsDirectory().set(definitionsDir); + } t.getReferencesFile().set(collectTask.get().getOutputFile()); }); project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(validateTask)); From 1e9a4ec3d667c881bdea5ab3ca15c200e2025851 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 1 Aug 2025 10:53:49 -0700 Subject: [PATCH 22/22] guard global plugin too --- .../transport/GlobalTransportVersionManagementPlugin.java | 6 +++++- .../transport/ValidateTransportVersionDefinitionsTask.java | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index 622c1aa2a349c..c4e4c7d033e79 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -13,6 +13,7 @@ import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.dsl.DependencyHandler; +import org.gradle.api.file.Directory; import org.gradle.api.plugins.JavaPlugin; import org.gradle.api.tasks.Copy; import org.gradle.language.base.plugins.LifecycleBasePlugin; @@ -42,7 +43,10 @@ public void apply(Project project) { .register("validateTransportVersionDefinitions", ValidateTransportVersionDefinitionsTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Validates that all defined TransportVersion constants are used in at least one project"); - t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); + Directory definitionsDir = TransportVersionUtils.getDefinitionsDirectory(project); + if (definitionsDir.getAsFile().exists()) { + t.getDefinitionsDirectory().set(definitionsDir); + } t.getReferencesFiles().setFrom(tvReferencesConfig); }); project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(validateTask)); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java index 65941f1a245bc..1c55bacb31e8f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java @@ -16,6 +16,7 @@ import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.InputDirectory; import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.Optional; import org.gradle.api.tasks.PathSensitive; import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskAction; @@ -36,6 +37,7 @@ public abstract class ValidateTransportVersionDefinitionsTask extends DefaultTask { @InputDirectory + @Optional @PathSensitive(PathSensitivity.RELATIVE) public abstract DirectoryProperty getDefinitionsDirectory();