From 7dcf81b0f82a45ac07a1fd8237409da314d7dcc7 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Wed, 13 Aug 2025 12:10:07 -0700 Subject: [PATCH 01/68] initial draft --- ...enerateTransportVersionDefinitionTask.java | 241 ++++++++++++++++++ .../transport/TransportVersionUtils.java | 138 ++++++++-- 2 files changed, 353 insertions(+), 26 deletions(-) create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java new file mode 100644 index 0000000000000..b7bb6ac8f8235 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -0,0 +1,241 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.elasticsearch.gradle.Version; +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TransportVersionDefinition; +import org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TransportVersionLatest; +import org.gradle.api.DefaultTask; +import org.gradle.api.Project; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.provider.ListProperty; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.Optional; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; +import org.gradle.api.tasks.TaskAction; +import org.gradle.api.tasks.options.Option; +import org.gradle.process.ExecOperations; +import org.gradle.process.ExecResult; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.function.BiFunction; +import java.util.function.Function; + +import javax.inject.Inject; + +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.IdComponents.PATCH; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.IdComponents.SERVER; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.IdComponents.SUBSIDIARY; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.LATEST_DIR; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.SERVERLESS_BRANCH; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TransportVersionId; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.latestFilePath; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readLatestFile; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.writeDefinitionFile; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.writeLatestFile; + +/** + * This task generates transport version definition files. These files + * are runtime resources that TransportVersion loads statically. + * They contain a comma separated list of integer ids. Each file is named the same + * as the transport version name itself (with the .csv suffix). + */ +public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask { + + /** + * Specifies the directory in which contains all TransportVersionSet data files. + * + * @return + */ + @InputDirectory + // @Optional + @PathSensitive(PathSensitivity.RELATIVE) + public abstract DirectoryProperty getResourcesDirectory();// The plugin should always set this, not optional + + // assumption: this task is always run on main, so we can determine the name by diffing with main and looking for new files added in the + // definition directory. (not true: once we generate the file, this will no longer hold true if we then need to update it) + + /** + * Used to set the name of the TransportVersionSet for which a data file will be generated. + */ + @Input + // @Optional + @Option(option = "name", description = "TBD") + public abstract Property getTransportVersionName(); // The plugin should always set this, not optional + + /** + * Used to set the `major.minor` release version for which the specific TransportVersion ID will be generated. + * E.g.: "9.2", "8.18", etc. + */ + @Optional // This is optional as we will look for labels in an env var if this is not set. + @Input + @Option(option = "branches", description = "The branches for which to generate IDs, e.g. --branch=\"9.2,9.1,SERVERLESS\"") + public abstract ListProperty getBranches(); + + // @Optional + // @Input + // public abstract Property> getIdIncrementSupplier(); + + private final Path rootPath; + private final ExecOperations execOperations; + + @Inject + public GenerateTransportVersionDefinitionTask(Project project, ExecOperations execOperations) { + this.execOperations = execOperations; + this.rootPath = getProject().getRootProject().getLayout().getProjectDirectory().getAsFile().toPath(); + } + + @TaskAction + public void generateTransportVersionData() throws IOException { + getLogger().lifecycle("Name: " + getTransportVersionName().get()); + getLogger().lifecycle("Versions: " + getBranches().get()); + Path resourcesDir = Objects.requireNonNull(getResourcesDirectory().getAsFile().get()).toPath(); + String name = getTransportVersionName().isPresent() ? getTransportVersionName().get() : findLocalTransportVersionName(); + Set targetMinorVersions = new HashSet<>( + getBranches().isPresent() ? new HashSet<>(getBranches().get()) : findTargetMinorVersions() + ); + + List ids = new ArrayList<>(); + for (String branchName : getKnownBranchNames(resourcesDir)) { + TransportVersionLatest latest = readLatestFile(resourcesDir, branchName); + + if (name.equals(latest.name())) { + if (targetMinorVersions.contains(branchName) == false) { + // Regenerate to make this operation idempotent. Need to undo prior updates to the latest files if the list of minor + // versions has changed. + + // TODO should we just always regen? + writeLatestFile(resourcesDir, readExistingLatest(branchName)); + } + } else { + if (targetMinorVersions.contains(branchName)) { + // increment + var incrementedId = incrementTVId(latest.id(), branchName); + ids.add(incrementedId); + writeLatestFile(resourcesDir, new TransportVersionLatest(branchName, name, incrementedId)); + } + } + } + // (Re)write the definition file. + writeDefinitionFile(resourcesDir, new TransportVersionDefinition(name, ids)); + } + + private TransportVersionId incrementTVId(TransportVersionId id, String branchName) { + // We can only run this task on main, so the ElasticsearchVersion will be for main. + Version mainVersion = VersionProperties.getElasticsearchVersion(); + String latestFileNameForMain = mainVersion.getMajor() + "." + mainVersion.getMinor(); + + final var isMain = branchName.equals(latestFileNameForMain); + if (isMain) { + return id.bumpComponent(SERVER); + } else if (branchName.equals(SERVERLESS_BRANCH)) { + return id.bumpComponent(SUBSIDIARY); + } else { + return id.bumpComponent(PATCH); + } + } + + private Set getKnownBranchNames(Path resourcesDir) { + // list files under latest + // TODO add serverless? Otherwise just delete this function if it that doesn't make sense here. + return recordExistingResources(resourcesDir.resolve(LATEST_DIR)); + } + + private String findLocalTransportVersionName() { + // check for missing + // if none missing, look at git diff against main + return ""; + } + + private List findTargetMinorVersions() { + // look for env var indicating github PR link from CI + // use github api to find current labels, filter down to version labels + // map version labels to branches + return List.of(); + } + + // TODO duplicated + private Set recordExistingResources(Path resourcesPath) { + String output = gitCommand("ls-tree", "--name-only", "-r", "main", resourcesPath.toString()); + return Set.of(output.split(System.lineSeparator())); + } + + // TODO duplicated + private TransportVersionLatest readExistingLatest(String branch) { + return readExistingFile(branch, this::latestRelativePath, TransportVersionLatest::fromString); + } + + // TODO duplicated + private T readExistingFile(String name, Function pathFunction, BiFunction parser) { + String relativePath = pathFunction.apply(name); + String content = gitCommand("show", "main:" + relativePath).strip(); + return parser.apply(relativePath, content); + } + + // TODO duplicated + private String latestRelativePath(String branch) { + return relativePath(latestFilePath(getResourcesDirectory().get(), branch)); + } + + // TODO duplicated + private Path resourcesDirPath() { + return getResourcesDirectory().get().getAsFile().toPath(); + } + + // TODO duplicated + private String relativePath(Path file) { + return rootPath.relativize(file).toString(); + } + + // TODO duplicated + public String gitCommand(String... args) { + final ByteArrayOutputStream stdout = new ByteArrayOutputStream(); + + List command = new ArrayList<>(); + Collections.addAll(command, "git", "-C", rootPath.toAbsolutePath().toString()); + Collections.addAll(command, args); + + ExecResult result = execOperations.exec(spec -> { + spec.setCommandLine(command); + spec.setStandardOutput(stdout); + spec.setErrorOutput(stdout); + spec.setIgnoreExitValue(true); + }); + + if (result.getExitValue() != 0) { + throw new RuntimeException( + "git command failed with exit code " + + result.getExitValue() + + System.lineSeparator() + + "command: " + + String.join(" ", command) + + System.lineSeparator() + + "output:" + + System.lineSeparator() + + stdout.toString(StandardCharsets.UTF_8) + ); + } + + return stdout.toString(StandardCharsets.UTF_8); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 4e1cb86276634..1a1cbca8aa343 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -9,6 +9,9 @@ package org.elasticsearch.gradle.internal.transport; +import com.google.common.collect.Comparators; + +import org.gradle.api.GradleException; import org.gradle.api.Project; import org.gradle.api.attributes.Attribute; import org.gradle.api.attributes.AttributeContainer; @@ -19,14 +22,26 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.IdComponents.MAJOR; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.IdComponents.PATCH; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.IdComponents.SERVER; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.IdComponents.SUBSIDIARY; import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE; class TransportVersionUtils { - static final Attribute TRANSPORT_VERSION_REFERENCES_ATTRIBUTE = Attribute.of("transport-version-references", Boolean.class); + static final String LATEST_DIR = "latest"; + static final String DEFINED_DIR = "defined"; + static final String SERVERLESS_BRANCH = "SERVERLESS"; + + private static final String CSV_SUFFIX = ".csv"; + record TransportVersionReference(String name, String location) { @Override public String toString() { @@ -35,15 +50,22 @@ public String toString() { } record TransportVersionDefinition(String name, List ids) { + TransportVersionDefinition { + validateNameFormat(name); + if (Comparators.isInOrder(ids, Comparator.reverseOrder()) == false) { + throw new IllegalArgumentException("Ids are not in decreasing order"); + } + } + public static TransportVersionDefinition fromString(String filename, String contents) { assert filename.endsWith(".csv"); - String name = filename.substring(0, filename.length() - 4); + String name = filename.substring(0, filename.length() - CSV_SUFFIX.length()); List ids = new ArrayList<>(); if (contents.isEmpty() == false) { for (String rawId : contents.split(",")) { try { - ids.add(parseId(rawId)); + ids.add(TransportVersionId.fromString(rawId)); } catch (NumberFormatException e) { throw new IllegalStateException("Failed to parse id " + rawId + " in " + filename, e); } @@ -55,30 +77,74 @@ public static TransportVersionDefinition fromString(String filename, String cont } record TransportVersionLatest(String branch, String name, TransportVersionId id) { + TransportVersionLatest { + validateNameFormat(name); + validateBranchFormat(branch); + } + public static TransportVersionLatest fromString(String filename, String contents) { assert filename.endsWith(".csv"); - String branch = filename.substring(0, filename.length() - 4); + String branch = filename.substring(0, filename.length() - CSV_SUFFIX.length()); String[] parts = contents.split(","); if (parts.length != 2) { throw new IllegalStateException("Invalid transport version latest file [" + filename + "]: " + contents); } - return new TransportVersionLatest(branch, parts[0], parseId(parts[1])); + return new TransportVersionLatest(branch, parts[0], TransportVersionId.fromString(parts[1])); + } + } + + /** + * The TV format: + *

+ * MM_NNN_S_PP + *

+ * M - The major version of Elasticsearch + * NNN - The server version part + * S - The subsidiary version part. It should always be 0 here, it is only used in subsidiary repositories. + * PP - The patch version part + */ + public enum IdComponents { + MAJOR(1_000_0_00, 2), + SERVER(1_0_00, 3), + SUBSIDIARY(1_00, 1), + PATCH(1, 2); + + private final int value; + private final int max; + + IdComponents(int value, int numDigits) { + this.value = value; + this.max = (int) Math.pow(10, numDigits-1); } } record TransportVersionId(int complete, int major, int server, int subsidiary, int patch) implements Comparable { + public static TransportVersionId fromInt(int complete) { + int patch = complete % PATCH.value; + int subsidiary = (complete / SUBSIDIARY.value) % SUBSIDIARY.max; + int server = (complete / SERVER.value) % SERVER.max; + int major = complete / MAJOR.value; - static TransportVersionId fromString(String s) { - int complete = Integer.parseInt(s); - int patch = complete % 100; - int subsidiary = (complete / 100) % 10; - int server = (complete / 1000) % 1000; - int major = complete / 1000000; return new TransportVersionId(complete, major, server, subsidiary, patch); } + public TransportVersionId bumpComponent(IdComponents idComponents) { + int zeroesCleared = (complete / idComponents.value) * idComponents.value; + int newId = zeroesCleared + idComponents.value; + if ((newId / idComponents.value) % idComponents.max == 0) { + throw new IllegalStateException( + "Insufficient" + idComponents.name() + " version section in TransportVersion: " + complete + ", Cannot bump." + ); + } + return fromInt(newId); + } + + public static TransportVersionId fromString(String filename) { + return fromInt(Integer.parseInt(filename)); + } + @Override public int compareTo(TransportVersionId o) { return Integer.compare(complete, o.complete); @@ -95,11 +161,11 @@ public int base() { } static Path definitionFilePath(Directory resourcesDirectory, String name) { - return getDefinitionsDirectory(resourcesDirectory).getAsFile().toPath().resolve(name + ".csv"); + return getDefinitionsDirectory(resourcesDirectory).getAsFile().toPath().resolve(name + CSV_SUFFIX); } - static Path latestFilePath(Directory resourcesDirectory, String name) { - return getLatestDirectory(resourcesDirectory).getAsFile().toPath().resolve(name + ".csv"); + static Path latestFilePath(Directory resourcesDir, String name) { + return getLatestDirectory(resourcesDir).getAsFile().toPath().resolve(name + CSV_SUFFIX); } static TransportVersionDefinition readDefinitionFile(Path file) throws IOException { @@ -107,11 +173,41 @@ static TransportVersionDefinition readDefinitionFile(Path file) throws IOExcepti return TransportVersionDefinition.fromString(file.getFileName().toString(), contents); } + static void writeDefinitionFile(Path resourcesDir, TransportVersionDefinition definition) throws IOException { + Files.writeString( + resourcesDir.resolve(DEFINED_DIR).resolve(definition.name() + CSV_SUFFIX), + definition.ids().stream().map(id -> String.valueOf(id.complete())).collect(Collectors.joining(",")) + "\n", + StandardCharsets.UTF_8 + ); + } + static TransportVersionLatest readLatestFile(Path file) throws IOException { String contents = Files.readString(file, StandardCharsets.UTF_8).strip(); return TransportVersionLatest.fromString(file.getFileName().toString(), contents); } + static TransportVersionLatest readLatestFile(Path resourcesDir, String branchName) throws IOException { + Path latestFilePath = resourcesDir.resolve(LATEST_DIR).resolve(branchName + CSV_SUFFIX); + return readLatestFile(latestFilePath); + } + + static void writeLatestFile(Path resourcesDir, TransportVersionLatest latest) throws IOException { + var path = resourcesDir.resolve(LATEST_DIR).resolve(latest.branch + CSV_SUFFIX); + Files.writeString(path, latest.name() + "," + latest.id().complete + "\n", StandardCharsets.UTF_8); + } + + static void validateNameFormat(String name) { + if (Pattern.compile("^\\w+$").matcher(name).matches() == false) { + throw new GradleException("The TransportVersion name must only contain underscores and alphanumeric characters: " + name); + } + } + + static void validateBranchFormat(String branchName) { + if (Pattern.compile("^(\\d\\.\\d)|SERVERLESS$").matcher(branchName).matches() == false) { + throw new GradleException("The branch name must be of the form \"int.int\" or \"SERVERLESS\""); + } + } + static List readReferencesFile(Path file) throws IOException { assert file.endsWith(".txt"); List results = new ArrayList<>(); @@ -125,21 +221,12 @@ static List readReferencesFile(Path file) throws IOEx return results; } - private static TransportVersionId parseId(String rawId) { - int complete = Integer.parseInt(rawId); - int patch = complete % 100; - int subsidiary = (complete / 100) % 10; - int server = (complete / 1000) % 1000; - int major = complete / 1000000; - return new TransportVersionId(complete, major, server, subsidiary, patch); - } - static Directory getDefinitionsDirectory(Directory resourcesDirectory) { - return resourcesDirectory.dir("defined"); + return resourcesDirectory.dir(DEFINED_DIR); } static Directory getLatestDirectory(Directory resourcesDirectory) { - return resourcesDirectory.dir("latest"); + return resourcesDirectory.dir(LATEST_DIR); } static Directory getResourcesDirectory(Project project) { @@ -155,5 +242,4 @@ static void addTransportVersionReferencesAttribute(AttributeContainer attributes attributes.attribute(ARTIFACT_TYPE_ATTRIBUTE, "txt"); attributes.attribute(TransportVersionUtils.TRANSPORT_VERSION_REFERENCES_ATTRIBUTE, true); } - } From 4e1b998c7f2d8ac11bbc9703fb5adaadd63661b0 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 13 Aug 2025 19:18:19 +0000 Subject: [PATCH 02/68] [CI] Auto commit changes from spotless --- .../gradle/internal/transport/TransportVersionUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 1a1cbca8aa343..65e999e33b3e1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -116,7 +116,7 @@ public enum IdComponents { IdComponents(int value, int numDigits) { this.value = value; - this.max = (int) Math.pow(10, numDigits-1); + this.max = (int) Math.pow(10, numDigits - 1); } } From aea402948dd9d4329383ff8a1083a9a260510f19 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Wed, 13 Aug 2025 12:38:32 -0700 Subject: [PATCH 03/68] fix bug --- .../gradle/internal/transport/TransportVersionUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 1a1cbca8aa343..79173a49f9e0e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -116,13 +116,13 @@ public enum IdComponents { IdComponents(int value, int numDigits) { this.value = value; - this.max = (int) Math.pow(10, numDigits-1); + this.max = (int) Math.pow(10, numDigits); } } record TransportVersionId(int complete, int major, int server, int subsidiary, int patch) implements Comparable { public static TransportVersionId fromInt(int complete) { - int patch = complete % PATCH.value; + int patch = complete % PATCH.max; int subsidiary = (complete / SUBSIDIARY.value) % SUBSIDIARY.max; int server = (complete / SERVER.value) % SERVER.max; int major = complete / MAJOR.value; From 36813694c0877ffcfc3cd75123a2b3c80900975a Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Wed, 13 Aug 2025 14:43:07 -0700 Subject: [PATCH 04/68] fix bugs --- .../internal/transport/TransportVersionUtils.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 79173a49f9e0e..55cc20d3a6ea8 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -58,8 +58,9 @@ record TransportVersionDefinition(String name, List ids) { } public static TransportVersionDefinition fromString(String filename, String contents) { - assert filename.endsWith(".csv"); - String name = filename.substring(0, filename.length() - CSV_SUFFIX.length()); + assert filename.endsWith(CSV_SUFFIX); + String fileWithoutPath = Path.of(filename).getFileName().toString(); + String name = fileWithoutPath.substring(0, fileWithoutPath.length() - CSV_SUFFIX.length()); List ids = new ArrayList<>(); if (contents.isEmpty() == false) { @@ -84,7 +85,8 @@ record TransportVersionLatest(String branch, String name, TransportVersionId id) public static TransportVersionLatest fromString(String filename, String contents) { assert filename.endsWith(".csv"); - String branch = filename.substring(0, filename.length() - CSV_SUFFIX.length()); + String fileWithoutPath = Path.of(filename).getFileName().toString(); + String branch = fileWithoutPath.substring(0, fileWithoutPath.length() - CSV_SUFFIX.length()); String[] parts = contents.split(","); if (parts.length != 2) { @@ -203,13 +205,13 @@ static void validateNameFormat(String name) { } static void validateBranchFormat(String branchName) { - if (Pattern.compile("^(\\d\\.\\d)|SERVERLESS$").matcher(branchName).matches() == false) { + if (Pattern.compile("^(\\d+\\.\\d+)|SERVERLESS$").matcher(branchName).matches() == false) { throw new GradleException("The branch name must be of the form \"int.int\" or \"SERVERLESS\""); } } static List readReferencesFile(Path file) throws IOException { - assert file.endsWith(".txt"); + assert file.toString().endsWith(".txt"); List results = new ArrayList<>(); for (String line : Files.readAllLines(file, StandardCharsets.UTF_8)) { String[] parts = line.split(",", 2); From fc0e10632a7477703abdb8d1b3eb92d62f5547e7 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Wed, 13 Aug 2025 14:55:50 -0700 Subject: [PATCH 05/68] remove extra validation that was failing tests --- .../gradle/internal/transport/TransportVersionUtils.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 55cc20d3a6ea8..eca06bd849e03 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -52,9 +52,6 @@ public String toString() { record TransportVersionDefinition(String name, List ids) { TransportVersionDefinition { validateNameFormat(name); - if (Comparators.isInOrder(ids, Comparator.reverseOrder()) == false) { - throw new IllegalArgumentException("Ids are not in decreasing order"); - } } public static TransportVersionDefinition fromString(String filename, String contents) { From 4ad2c997d256066ae65065e37185e11ce96436b1 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 13 Aug 2025 22:02:34 +0000 Subject: [PATCH 06/68] [CI] Auto commit changes from spotless --- .../gradle/internal/transport/TransportVersionUtils.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index eca06bd849e03..0ad5d15b792b1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -9,8 +9,6 @@ package org.elasticsearch.gradle.internal.transport; -import com.google.common.collect.Comparators; - import org.gradle.api.GradleException; import org.gradle.api.Project; import org.gradle.api.attributes.Attribute; @@ -22,7 +20,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; -import java.util.Comparator; import java.util.List; import java.util.regex.Pattern; import java.util.stream.Collectors; From 9c89556c782f34f646c016a1ace938b54882d96d Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 13 Aug 2025 23:39:21 +0000 Subject: [PATCH 07/68] [CI] Auto commit changes from spotless --- .../gradle/internal/transport/TransportVersionId.java | 2 +- .../gradle/internal/transport/TransportVersionUtils.java | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionId.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionId.java index df0974ad4f977..32d91b549850c 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionId.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionId.java @@ -54,7 +54,7 @@ public TransportVersionId bumpComponent(Component component) { int newId = zeroesCleared + component.value; if ((newId / component.value) % component.max == 0) { throw new IllegalStateException( - "Insufficient" + component.name() + " version section in TransportVersion: " + complete + ", Cannot bump." + "Insufficient" + component.name() + " version section in TransportVersion: " + complete + ", Cannot bump." ); } return fromInt(newId); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 0facf004ab49e..78e58d0f529f6 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -17,8 +17,6 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; import java.util.regex.Pattern; import java.util.stream.Collectors; From dea290b4247d95b9c8371c19dcbe3dd23d551f43 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 14 Aug 2025 14:10:37 -0700 Subject: [PATCH 08/68] generate iter --- ...enerateTransportVersionDefinitionTask.java | 172 +++++++++++++----- .../transport/TransportVersionLatest.java | 2 +- .../transport/TransportVersionUtils.java | 18 +- ...ValidateTransportVersionResourcesTask.java | 18 +- 4 files changed, 145 insertions(+), 65 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index eaf1b2416685d..ba6d077c95f02 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -9,15 +9,15 @@ package org.elasticsearch.gradle.internal.transport; -import org.elasticsearch.gradle.Version; -import org.elasticsearch.gradle.VersionProperties; import org.gradle.api.DefaultTask; import org.gradle.api.Project; +import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.DirectoryProperty; import org.gradle.api.provider.ListProperty; import org.gradle.api.provider.Property; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.Optional; import org.gradle.api.tasks.PathSensitive; import org.gradle.api.tasks.PathSensitivity; @@ -27,8 +27,10 @@ import org.gradle.process.ExecResult; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; @@ -38,14 +40,12 @@ import java.util.Set; import java.util.function.BiFunction; import java.util.function.Function; +import java.util.stream.Stream; import javax.inject.Inject; -import static org.elasticsearch.gradle.internal.transport.TransportVersionId.Component.PATCH; -import static org.elasticsearch.gradle.internal.transport.TransportVersionId.Component.SERVER; -import static org.elasticsearch.gradle.internal.transport.TransportVersionId.Component.SUBSIDIARY; +import static org.elasticsearch.gradle.internal.transport.TransportVersionReference.listFromFile; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.LATEST_DIR; -import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.SERVERLESS_BRANCH; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.latestFilePath; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readLatestFile; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.writeDefinitionFile; @@ -65,10 +65,13 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask * @return */ @InputDirectory - // @Optional @PathSensitive(PathSensitivity.RELATIVE) public abstract DirectoryProperty getResourcesDirectory();// The plugin should always set this, not optional + @InputFiles + @PathSensitive(PathSensitivity.RELATIVE) + public abstract ConfigurableFileCollection getReferencesFiles(); + // assumption: this task is always run on main, so we can determine the name by diffing with main and looking for new files added in the // definition directory. (not true: once we generate the file, this will no longer hold true if we then need to update it) @@ -76,7 +79,7 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask * Used to set the name of the TransportVersionSet for which a data file will be generated. */ @Input - // @Optional + @Optional @Option(option = "name", description = "TBD") public abstract Property getTransportVersionName(); // The plugin should always set this, not optional @@ -86,9 +89,19 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask */ @Optional // This is optional as we will look for labels in an env var if this is not set. @Input - @Option(option = "branches", description = "The branches for which to generate IDs, e.g. --branch=\"9.2,9.1,SERVERLESS\"") + @Option(option = "branches", description = "The branches for which to generate IDs, e.g. --releaseBranch=\"main,9.1\"") public abstract ListProperty getBranches(); + @Input + @Optional + @Option(option = "increment", description = "TBD") + public abstract Property getPrimaryIncrement(); + + @Input + public abstract Property getMainReleaseBranch(); + + @Input + public abstract Property getResourcesProjectDir(); // @Optional // @Input // public abstract Property> getIdIncrementSupplier(); @@ -103,68 +116,127 @@ public GenerateTransportVersionDefinitionTask(Project project, ExecOperations ex } @TaskAction - public void generateTransportVersionData() throws IOException { + public void run() throws IOException { getLogger().lifecycle("Name: " + getTransportVersionName().get()); getLogger().lifecycle("Versions: " + getBranches().get()); Path resourcesDir = Objects.requireNonNull(getResourcesDirectory().getAsFile().get()).toPath(); - String name = getTransportVersionName().isPresent() ? getTransportVersionName().get() : findLocalTransportVersionName(); - Set targetMinorVersions = new HashSet<>( - getBranches().isPresent() ? new HashSet<>(getBranches().get()) : findTargetMinorVersions() - ); + Set referencedNames = getReferencedNames(); + List changedDefinitionNames = getChangedDefinitionNames(); + String name = getTransportVersionName().isPresent() ? getTransportVersionName().get() : findAddedTransportVersionName(referencedNames, changedDefinitionNames); + if (name.isEmpty()) { + resetAllLatestFiles(resourcesDir); + } else { + List ids = updateLatestFiles(resourcesDir, name); + // (Re)write the definition file. + writeDefinitionFile(resourcesDir, new TransportVersionDefinition(name, ids)); + } + + removeUnusedDefinitions(referencedNames, changedDefinitionNames); + } + + private Set getReferencedNames() throws IOException{ + Set referencedNames = new HashSet<>(); + for (var referencesFile : getReferencesFiles()) { + listFromFile(referencesFile.toPath()).stream().map(TransportVersionReference::name).forEach(referencedNames::add); + } + return referencedNames; + } + + private List updateLatestFiles(Path resourcesDir, String name) throws IOException { + Set targetReleaseBranches = new HashSet<>( + getBranches().isPresent() ? mapBranchesToReleaseBranches(getBranches().get()) : findTargetReleaseBranches() + ); + String mainReleaseBranch = getMainReleaseBranch().get(); + int primaryIncrement = getPrimaryIncrement().get(); List ids = new ArrayList<>(); - for (String branchName : getKnownBranchNames(resourcesDir)) { - TransportVersionLatest latest = readLatestFile(resourcesDir, branchName); + + // TODO: this should be reading from main, in case there are merge conflicts that have messed up the latest files + for (TransportVersionLatest latest : getKnownLatestFiles(resourcesDir)) { if (name.equals(latest.name())) { - if (targetMinorVersions.contains(branchName) == false) { + if (targetReleaseBranches.contains(latest.releaseBranch()) == false) { + // we don't want to target this latest file but we already changed it // Regenerate to make this operation idempotent. Need to undo prior updates to the latest files if the list of minor // versions has changed. - - // TODO should we just always regen? - writeLatestFile(resourcesDir, readExistingLatest(branchName)); + writeLatestFile(resourcesDir, readLatestFromMain(latest.releaseBranch())); } } else { - if (targetMinorVersions.contains(branchName)) { - // increment - var incrementedId = incrementTVId(latest.id(), branchName); - ids.add(incrementedId); - writeLatestFile(resourcesDir, new TransportVersionLatest(branchName, name, incrementedId)); + if (targetReleaseBranches.contains(latest.releaseBranch())) { + int increment = latest.releaseBranch().equals(mainReleaseBranch) ? primaryIncrement : 1; + TransportVersionId id = TransportVersionId.fromInt(latest.id().complete() + increment); + ids.add(id); + writeLatestFile(resourcesDir, new TransportVersionLatest(latest.releaseBranch(), name, id)); } } } - // (Re)write the definition file. - writeDefinitionFile(resourcesDir, new TransportVersionDefinition(name, ids)); + + return ids; } - private TransportVersionId incrementTVId(TransportVersionId id, String branchName) { - // We can only run this task on main, so the ElasticsearchVersion will be for main. - Version mainVersion = VersionProperties.getElasticsearchVersion(); - String latestFileNameForMain = mainVersion.getMajor() + "." + mainVersion.getMinor(); + private void resetAllLatestFiles(Path resourcesDir) throws IOException { + for (TransportVersionLatest latest : getKnownLatestFiles(resourcesDir)) { + writeLatestFile(resourcesDir, readLatestFromMain(latest.releaseBranch())); + } + } - final var isMain = branchName.equals(latestFileNameForMain); - if (isMain) { - return id.bumpComponent(SERVER); - } else if (branchName.equals(SERVERLESS_BRANCH)) { - return id.bumpComponent(SUBSIDIARY); - } else { - return id.bumpComponent(PATCH); + private List getChangedDefinitionNames() throws IOException { + List changedDefinitionNames = new ArrayList<>(); + String namedDefinitionsBasePath = TransportVersionUtils.getResourcePath(getResourcesProjectDir().get(), "definitions/named/"); + for (String changedResource : getChangedResources(namedDefinitionsBasePath)) { + if (changedResource.startsWith(namedDefinitionsBasePath)) { + String definitionName = changedResource.substring(changedResource.lastIndexOf(File.pathSeparator) + 1, changedResource.length() - 4 /* .csv */); + changedDefinitionNames.add(definitionName); + } + } + return changedDefinitionNames; + } + + private void removeUnusedDefinitions(Set referencedNames, List changedDefinitionNames) throws IOException { + for (String definitionName : changedDefinitionNames) { + if (referencedNames.contains(definitionName) == false) { + // we added this definition file, but it's now unreferenced, so delete it + getLogger().lifecycle("Deleting unreferenced named transport version definition [" + definitionName + "]"); + Path definitionPath = TransportVersionUtils.definitionFilePath(getResourcesDirectory().get(), definitionName); + Files.delete(definitionPath); + } } } - private Set getKnownBranchNames(Path resourcesDir) { - // list files under latest - // TODO add serverless? Otherwise just delete this function if it that doesn't make sense here. - return recordExistingResources(resourcesDir.resolve(LATEST_DIR)); + private List mapBranchesToReleaseBranches(List strings) { + return strings.stream().map(branch -> branch.equals("main") ? getMainReleaseBranch().get() : branch).toList(); } - private String findLocalTransportVersionName() { - // check for missing - // if none missing, look at git diff against main - return ""; + private List getKnownLatestFiles(Path resourcesDir) throws IOException{ + List latestFiles = new ArrayList<>(); + try (Stream stream = Files.list(resourcesDir.resolve(LATEST_DIR))) { + for (Path latestFile : stream.toList()) { + latestFiles.add(readLatestFile(latestFile)); + } + } + return latestFiles; + } + + private String findAddedTransportVersionName(Set referencedNames, List changedDefinitionNames) throws IOException { + + // First check for unreferenced names. There should only be at most one. If there is more than + // one reference the remaining reference will still be unreferenced when validation runs later + // and the developer will remove the unreferenced name. + for (String referencedName : referencedNames) { + Path definitionFile = TransportVersionUtils.definitionFilePath(getResourcesDirectory().get(), referencedName); + if (Files.exists(definitionFile) == false) { + return referencedName; + } + } + + if (changedDefinitionNames.isEmpty()) { + return ""; + } else { + return changedDefinitionNames.getFirst(); + } } - private List findTargetMinorVersions() { + private List findTargetReleaseBranches() { // look for env var indicating github PR link from CI // use github api to find current labels, filter down to version labels // map version labels to branches @@ -172,13 +244,13 @@ private List findTargetMinorVersions() { } // TODO duplicated - private Set recordExistingResources(Path resourcesPath) { - String output = gitCommand("ls-tree", "--name-only", "-r", "main", resourcesPath.toString()); + private Set getChangedResources(String subPath) { + String output = gitCommand("ls-tree", "--name-only", "-r", "main", getResourcesProjectDir().get() + "/" + subPath); return Set.of(output.split(System.lineSeparator())); } // TODO duplicated - private TransportVersionLatest readExistingLatest(String branch) { + private TransportVersionLatest readLatestFromMain(String branch) { return readExistingFile(branch, this::latestRelativePath, TransportVersionLatest::fromString); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionLatest.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionLatest.java index 2818b74d52937..d1f1af543a29f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionLatest.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionLatest.java @@ -9,7 +9,7 @@ package org.elasticsearch.gradle.internal.transport; -record TransportVersionLatest(String branch, String name, TransportVersionId id) { +record TransportVersionLatest(String releaseBranch, String name, TransportVersionId id) { public static TransportVersionLatest fromString(String filename, String contents) { assert filename.endsWith(".csv"); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 78e58d0f529f6..7a5e7c1e6db28 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -13,6 +13,7 @@ import org.gradle.api.Project; import org.gradle.api.file.Directory; +import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -23,7 +24,6 @@ class TransportVersionUtils { static final String LATEST_DIR = "latest"; static final String DEFINED_DIR = "defined"; - static final String SERVERLESS_BRANCH = "SERVERLESS"; private static final String CSV_SUFFIX = ".csv"; @@ -53,13 +53,13 @@ static TransportVersionLatest readLatestFile(Path file) throws IOException { return TransportVersionLatest.fromString(file.getFileName().toString(), contents); } - static TransportVersionLatest readLatestFile(Path resourcesDir, String branchName) throws IOException { - Path latestFilePath = resourcesDir.resolve(LATEST_DIR).resolve(branchName + CSV_SUFFIX); + static TransportVersionLatest readLatestFile(Path resourcesDir, String latestName) throws IOException { + Path latestFilePath = resourcesDir.resolve(LATEST_DIR).resolve(latestName + CSV_SUFFIX); return readLatestFile(latestFilePath); } static void writeLatestFile(Path resourcesDir, TransportVersionLatest latest) throws IOException { - var path = resourcesDir.resolve(LATEST_DIR).resolve(latest.branch() + CSV_SUFFIX); + var path = resourcesDir.resolve(LATEST_DIR).resolve(latest.releaseBranch() + CSV_SUFFIX); Files.writeString(path, latest.name() + "," + latest.id().complete() + "\n", StandardCharsets.UTF_8); } @@ -71,7 +71,7 @@ static void validateNameFormat(String name) { static void validateBranchFormat(String branchName) { if (Pattern.compile("^(\\d+\\.\\d+)|SERVERLESS$").matcher(branchName).matches() == false) { - throw new GradleException("The branch name must be of the form \"int.int\" or \"SERVERLESS\""); + throw new GradleException("The releaseBranch name must be of the form \"int.int\" or \"SERVERLESS\""); } } @@ -83,6 +83,10 @@ static Directory getLatestDirectory(Directory resourcesDirectory) { return resourcesDirectory.dir(LATEST_DIR); } + static String getResourcePath(String resourcesProjectPath, String subPath) { + return resourcesProjectPath + "/src/main/resources/transport/" + subPath; + } + static Directory getResourcesDirectory(Project project) { var projectName = project.findProperty("org.elasticsearch.transport.definitionsProject"); if (projectName == null) { @@ -91,4 +95,8 @@ static Directory getResourcesDirectory(Project project) { Directory projectDir = project.project(projectName.toString()).getLayout().getProjectDirectory(); return projectDir.dir("src/main/resources/transport"); } + + static String getNameFromCsv(String filepath) { + return filepath.substring(filepath.lastIndexOf(File.pathSeparator) + 1, filepath.length() - 4 /* .csv */); + } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java index 2791aca9e323f..df5e2d2a91e0e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java @@ -81,7 +81,7 @@ private record IdAndDefinition(TransportVersionId id, TransportVersionDefinition private final Map definedIds = new HashMap<>(); // lookup of base ids back to definition private final Map> idsByBase = new HashMap<>(); - // direct lookup of latest for each branch + // direct lookup of latest for each releaseBranch Map latestByBranch = new HashMap<>(); @Inject @@ -210,7 +210,7 @@ private void recordAndValidateDefinition(TransportVersionDefinition definition) } if (ndx == 0) { - // TODO: initial versions will only be applicable to a release branch, so they won't have an associated + // TODO: initial versions will only be applicable to a release releaseBranch, so they won't have an associated // main version. They will also be loaded differently in the future, but until they are separate, we ignore them here. if (id.patch() != 0 && definition.name().startsWith("initial_") == false) { throwDefinitionFailure(definition.name(), "has patch version " + id.complete() + " as primary id"); @@ -221,7 +221,7 @@ private void recordAndValidateDefinition(TransportVersionDefinition definition) } } - // check modifications of ids on same branch, ie sharing same base + // check modifications of ids on same releaseBranch, ie sharing same base TransportVersionId maybeModifiedId = existingIdsByBase.get(id.base()); if (maybeModifiedId != null && maybeModifiedId.complete() != id.complete()) { throwDefinitionFailure(definition.name(), "modifies existing patch id from " + maybeModifiedId + " to " + id); @@ -247,15 +247,15 @@ private T readExistingFile(String name, Function pathFunctio } private void recordAndValidateLatest(TransportVersionLatest latest) { - latestByBranch.put(latest.branch(), latest); + latestByBranch.put(latest.releaseBranch(), latest); TransportVersionDefinition latestDefinition = definitions.get(latest.name()); if (latestDefinition == null) { - throwLatestFailure(latest.branch(), "contains transport version name [" + latest.name() + "] which is not defined"); + throwLatestFailure(latest.releaseBranch(), "contains transport version name [" + latest.name() + "] which is not defined"); } if (latestDefinition.ids().contains(latest.id()) == false) { throwLatestFailure( - latest.branch(), + latest.releaseBranch(), "has id " + latest.id() + " which is not in definition [" + definitionRelativePath(latest.name()) + "]" ); } @@ -264,7 +264,7 @@ private void recordAndValidateLatest(TransportVersionLatest latest) { IdAndDefinition lastId = baseIds.getLast(); if (lastId.id().complete() != latest.id().complete()) { throwLatestFailure( - latest.branch(), + latest.releaseBranch(), "has id " + latest.id() + " from [" @@ -279,10 +279,10 @@ private void recordAndValidateLatest(TransportVersionLatest latest) { ); } - TransportVersionLatest existingLatest = readExistingLatest(latest.branch()); + TransportVersionLatest existingLatest = readExistingLatest(latest.releaseBranch()); if (existingLatest != null) { if (latest.id().patch() != 0 && latest.id().base() != existingLatest.id().base()) { - throwLatestFailure(latest.branch(), "modifies base id from " + existingLatest.id().base() + " to " + latest.id().base()); + throwLatestFailure(latest.releaseBranch(), "modifies base id from " + existingLatest.id().base() + " to " + latest.id().base()); } } } From d0beda59c324780caa3c1c29fa575fd45afcb0ec Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 14 Aug 2025 21:17:36 +0000 Subject: [PATCH 09/68] [CI] Auto commit changes from spotless --- .../GenerateTransportVersionDefinitionTask.java | 13 +++++++++---- .../ValidateTransportVersionResourcesTask.java | 5 ++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index ba6d077c95f02..7927a893c0d64 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -122,7 +122,9 @@ public void run() throws IOException { Path resourcesDir = Objects.requireNonNull(getResourcesDirectory().getAsFile().get()).toPath(); Set referencedNames = getReferencedNames(); List changedDefinitionNames = getChangedDefinitionNames(); - String name = getTransportVersionName().isPresent() ? getTransportVersionName().get() : findAddedTransportVersionName(referencedNames, changedDefinitionNames); + String name = getTransportVersionName().isPresent() + ? getTransportVersionName().get() + : findAddedTransportVersionName(referencedNames, changedDefinitionNames); if (name.isEmpty()) { resetAllLatestFiles(resourcesDir); @@ -135,7 +137,7 @@ public void run() throws IOException { removeUnusedDefinitions(referencedNames, changedDefinitionNames); } - private Set getReferencedNames() throws IOException{ + private Set getReferencedNames() throws IOException { Set referencedNames = new HashSet<>(); for (var referencesFile : getReferencesFiles()) { listFromFile(referencesFile.toPath()).stream().map(TransportVersionReference::name).forEach(referencedNames::add); @@ -185,7 +187,10 @@ private List getChangedDefinitionNames() throws IOException { String namedDefinitionsBasePath = TransportVersionUtils.getResourcePath(getResourcesProjectDir().get(), "definitions/named/"); for (String changedResource : getChangedResources(namedDefinitionsBasePath)) { if (changedResource.startsWith(namedDefinitionsBasePath)) { - String definitionName = changedResource.substring(changedResource.lastIndexOf(File.pathSeparator) + 1, changedResource.length() - 4 /* .csv */); + String definitionName = changedResource.substring( + changedResource.lastIndexOf(File.pathSeparator) + 1, + changedResource.length() - 4 /* .csv */ + ); changedDefinitionNames.add(definitionName); } } @@ -207,7 +212,7 @@ private List mapBranchesToReleaseBranches(List strings) { return strings.stream().map(branch -> branch.equals("main") ? getMainReleaseBranch().get() : branch).toList(); } - private List getKnownLatestFiles(Path resourcesDir) throws IOException{ + private List getKnownLatestFiles(Path resourcesDir) throws IOException { List latestFiles = new ArrayList<>(); try (Stream stream = Files.list(resourcesDir.resolve(LATEST_DIR))) { for (Path latestFile : stream.toList()) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java index df5e2d2a91e0e..9af7fe80f1afe 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java @@ -282,7 +282,10 @@ private void recordAndValidateLatest(TransportVersionLatest latest) { TransportVersionLatest existingLatest = readExistingLatest(latest.releaseBranch()); if (existingLatest != null) { if (latest.id().patch() != 0 && latest.id().base() != existingLatest.id().base()) { - throwLatestFailure(latest.releaseBranch(), "modifies base id from " + existingLatest.id().base() + " to " + latest.id().base()); + throwLatestFailure( + latest.releaseBranch(), + "modifies base id from " + existingLatest.id().base() + " to " + latest.id().base() + ); } } } From c5daf35cabde20a7f53fee88c7fde34de2074881 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 14 Aug 2025 16:39:56 -0700 Subject: [PATCH 10/68] more iter --- ...enerateTransportVersionDefinitionTask.java | 123 ++++++++++-------- 1 file changed, 69 insertions(+), 54 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index 7927a893c0d64..2b2daaaf82528 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -40,12 +40,11 @@ import java.util.Set; import java.util.function.BiFunction; import java.util.function.Function; -import java.util.stream.Stream; +import java.util.stream.Collectors; import javax.inject.Inject; import static org.elasticsearch.gradle.internal.transport.TransportVersionReference.listFromFile; -import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.LATEST_DIR; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.latestFilePath; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readLatestFile; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.writeDefinitionFile; @@ -84,12 +83,11 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask public abstract Property getTransportVersionName(); // The plugin should always set this, not optional /** - * Used to set the `major.minor` release version for which the specific TransportVersion ID will be generated. - * E.g.: "9.2", "8.18", etc. + * The release branch names the generated transport version should target. */ - @Optional // This is optional as we will look for labels in an env var if this is not set. + @Optional // In CI we find these from the github PR labels @Input - @Option(option = "branches", description = "The branches for which to generate IDs, e.g. --releaseBranch=\"main,9.1\"") + @Option(option = "branches", description = "The branches for which to generate IDs, e.g. --branches=\"main,9.1\"") public abstract ListProperty getBranches(); @Input @@ -102,9 +100,6 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask @Input public abstract Property getResourcesProjectDir(); - // @Optional - // @Input - // public abstract Property> getIdIncrementSupplier(); private final Path rootPath; private final ExecOperations execOperations; @@ -117,8 +112,6 @@ public GenerateTransportVersionDefinitionTask(Project project, ExecOperations ex @TaskAction public void run() throws IOException { - getLogger().lifecycle("Name: " + getTransportVersionName().get()); - getLogger().lifecycle("Versions: " + getBranches().get()); Path resourcesDir = Objects.requireNonNull(getResourcesDirectory().getAsFile().get()).toPath(); Set referencedNames = getReferencedNames(); List changedDefinitionNames = getChangedDefinitionNames(); @@ -134,7 +127,7 @@ public void run() throws IOException { writeDefinitionFile(resourcesDir, new TransportVersionDefinition(name, ids)); } - removeUnusedDefinitions(referencedNames, changedDefinitionNames); + removeUnusedNamedDefinitions(referencedNames, changedDefinitionNames); } private Set getReferencedNames() throws IOException { @@ -146,29 +139,26 @@ private Set getReferencedNames() throws IOException { } private List updateLatestFiles(Path resourcesDir, String name) throws IOException { - Set targetReleaseBranches = new HashSet<>( - getBranches().isPresent() ? mapBranchesToReleaseBranches(getBranches().get()) : findTargetReleaseBranches() - ); + Set targetReleaseBranches = getTargetReleaseBranches(); String mainReleaseBranch = getMainReleaseBranch().get(); int primaryIncrement = getPrimaryIncrement().get(); List ids = new ArrayList<>(); - // TODO: this should be reading from main, in case there are merge conflicts that have messed up the latest files - for (TransportVersionLatest latest : getKnownLatestFiles(resourcesDir)) { + for (TransportVersionLatest mainLatest : getMainLatestFiles()) { - if (name.equals(latest.name())) { - if (targetReleaseBranches.contains(latest.releaseBranch()) == false) { + if (name.equals(mainLatest.name())) { + if (targetReleaseBranches.contains(mainLatest.releaseBranch()) == false) { // we don't want to target this latest file but we already changed it // Regenerate to make this operation idempotent. Need to undo prior updates to the latest files if the list of minor // versions has changed. - writeLatestFile(resourcesDir, readLatestFromMain(latest.releaseBranch())); + writeLatestFile(resourcesDir, mainLatest); } } else { - if (targetReleaseBranches.contains(latest.releaseBranch())) { - int increment = latest.releaseBranch().equals(mainReleaseBranch) ? primaryIncrement : 1; - TransportVersionId id = TransportVersionId.fromInt(latest.id().complete() + increment); + if (targetReleaseBranches.contains(mainLatest.releaseBranch())) { + int increment = mainLatest.releaseBranch().equals(mainReleaseBranch) ? primaryIncrement : 1; + TransportVersionId id = TransportVersionId.fromInt(mainLatest.id().complete() + increment); ids.add(id); - writeLatestFile(resourcesDir, new TransportVersionLatest(latest.releaseBranch(), name, id)); + writeLatestFile(resourcesDir, new TransportVersionLatest(mainLatest.releaseBranch(), name, id)); } } } @@ -176,28 +166,64 @@ private List updateLatestFiles(Path resourcesDir, String nam return ids; } + private Set getTargetReleaseBranches() { + if (getBranches().isPresent()) { + return getBranches().get().stream().map(branch -> branch.equals("main") ? getMainReleaseBranch().get() : branch).collect(Collectors.toSet()); + } else { + // look for env var indicating github PR link from CI + // use github api to find current labels, filter down to version labels + // map version labels to branches + String prUrl = System.getenv("BUILDKITE_PULL_REQUEST"); + if (prUrl == null) { + throw new RuntimeException("When running outside CI, --branches must be specified"); + } + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + ExecResult result = execOperations.exec(spec -> { + spec.setCommandLine(List.of("gh", "pr", "view", prUrl, "--json", "labels", "--jq", ".labels[].name")); + spec.setErrorOutput(output); + spec.setStandardOutput(output); + spec.setIgnoreExitValue(true); + }); + if (result.getExitValue() != 0) { + throw new RuntimeException("Failed to get labels from github API:\n" + output); + } + Set targetReleaseBranches = new HashSet<>(); + for (String label : output.toString().split(System.lineSeparator())) { + if (label.startsWith("v") == false) { + continue; + } + int firstDot = label.indexOf('.'); + targetReleaseBranches.add(label.substring(1, label.indexOf('.', firstDot + 1))); + } + // if we didn't find any version labels we must be on serverless, so just use the main release branch + if (targetReleaseBranches.isEmpty()) { + targetReleaseBranches.add(getMainReleaseBranch().get()); + } + return targetReleaseBranches; + } + } + private void resetAllLatestFiles(Path resourcesDir) throws IOException { - for (TransportVersionLatest latest : getKnownLatestFiles(resourcesDir)) { - writeLatestFile(resourcesDir, readLatestFromMain(latest.releaseBranch())); + for (TransportVersionLatest latest : getMainLatestFiles()) { + writeLatestFile(resourcesDir, latest); } } private List getChangedDefinitionNames() throws IOException { List changedDefinitionNames = new ArrayList<>(); String namedDefinitionsBasePath = TransportVersionUtils.getResourcePath(getResourcesProjectDir().get(), "definitions/named/"); - for (String changedResource : getChangedResources(namedDefinitionsBasePath)) { - if (changedResource.startsWith(namedDefinitionsBasePath)) { - String definitionName = changedResource.substring( - changedResource.lastIndexOf(File.pathSeparator) + 1, - changedResource.length() - 4 /* .csv */ - ); - changedDefinitionNames.add(definitionName); - } + for (String mainResource : getMainResources(namedDefinitionsBasePath)) { + String definitionName = mainResource.substring( + mainResource.lastIndexOf(File.pathSeparator) + 1, + mainResource.length() - 4 /* .csv */ + ); + changedDefinitionNames.add(definitionName); } return changedDefinitionNames; } - private void removeUnusedDefinitions(Set referencedNames, List changedDefinitionNames) throws IOException { + private void removeUnusedNamedDefinitions(Set referencedNames, List changedDefinitionNames) throws IOException { for (String definitionName : changedDefinitionNames) { if (referencedNames.contains(definitionName) == false) { // we added this definition file, but it's now unreferenced, so delete it @@ -208,16 +234,12 @@ private void removeUnusedDefinitions(Set referencedNames, List c } } - private List mapBranchesToReleaseBranches(List strings) { - return strings.stream().map(branch -> branch.equals("main") ? getMainReleaseBranch().get() : branch).toList(); - } - - private List getKnownLatestFiles(Path resourcesDir) throws IOException { + private List getMainLatestFiles() throws IOException { List latestFiles = new ArrayList<>(); - try (Stream stream = Files.list(resourcesDir.resolve(LATEST_DIR))) { - for (Path latestFile : stream.toList()) { - latestFiles.add(readLatestFile(latestFile)); - } + String latestBasePath = TransportVersionUtils.getResourcePath(getResourcesProjectDir().get(), "latest/"); + for (String latestPath : getMainResources(latestBasePath)) { + TransportVersionLatest latest = readExistingFile(latestPath, this::latestRelativePath, TransportVersionLatest::fromString); + latestFiles.add(latest); } return latestFiles; } @@ -241,22 +263,15 @@ private String findAddedTransportVersionName(Set referencedNames, List findTargetReleaseBranches() { - // look for env var indicating github PR link from CI - // use github api to find current labels, filter down to version labels - // map version labels to branches - return List.of(); - } - // TODO duplicated - private Set getChangedResources(String subPath) { + private Set getMainResources(String subPath) { String output = gitCommand("ls-tree", "--name-only", "-r", "main", getResourcesProjectDir().get() + "/" + subPath); return Set.of(output.split(System.lineSeparator())); } // TODO duplicated - private TransportVersionLatest readLatestFromMain(String branch) { - return readExistingFile(branch, this::latestRelativePath, TransportVersionLatest::fromString); + private TransportVersionLatest readLatestFromMain(String releaseBranch) { + return readExistingFile(releaseBranch, this::latestRelativePath, TransportVersionLatest::fromString); } // TODO duplicated From 0c1c962466653fd624f5e8c8cd52ad31951ccb1d Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 14 Aug 2025 16:41:23 -0700 Subject: [PATCH 11/68] spotless --- .../transport/GenerateTransportVersionDefinitionTask.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index 2b2daaaf82528..5f6bef23ee6c3 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -46,7 +46,6 @@ import static org.elasticsearch.gradle.internal.transport.TransportVersionReference.listFromFile; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.latestFilePath; -import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readLatestFile; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.writeDefinitionFile; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.writeLatestFile; @@ -168,7 +167,10 @@ private List updateLatestFiles(Path resourcesDir, String nam private Set getTargetReleaseBranches() { if (getBranches().isPresent()) { - return getBranches().get().stream().map(branch -> branch.equals("main") ? getMainReleaseBranch().get() : branch).collect(Collectors.toSet()); + return getBranches().get() + .stream() + .map(branch -> branch.equals("main") ? getMainReleaseBranch().get() : branch) + .collect(Collectors.toSet()); } else { // look for env var indicating github PR link from CI // use github api to find current labels, filter down to version labels @@ -193,7 +195,7 @@ private Set getTargetReleaseBranches() { if (label.startsWith("v") == false) { continue; } - int firstDot = label.indexOf('.'); + int firstDot = label.indexOf('.'); targetReleaseBranches.add(label.substring(1, label.indexOf('.', firstDot + 1))); } // if we didn't find any version labels we must be on serverless, so just use the main release branch From c947d6aa4fcc086139384d1e56e406e7ed9a3679 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 14 Aug 2025 16:58:03 -0700 Subject: [PATCH 12/68] separate method for changed vs all main resources --- ...enerateTransportVersionDefinitionTask.java | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index 5f6bef23ee6c3..5673b446a0809 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -215,10 +215,10 @@ private void resetAllLatestFiles(Path resourcesDir) throws IOException { private List getChangedDefinitionNames() throws IOException { List changedDefinitionNames = new ArrayList<>(); String namedDefinitionsBasePath = TransportVersionUtils.getResourcePath(getResourcesProjectDir().get(), "definitions/named/"); - for (String mainResource : getMainResources(namedDefinitionsBasePath)) { - String definitionName = mainResource.substring( - mainResource.lastIndexOf(File.pathSeparator) + 1, - mainResource.length() - 4 /* .csv */ + for (String changedResource : getChangedResources(namedDefinitionsBasePath)) { + String definitionName = changedResource.substring( + changedResource.lastIndexOf(File.pathSeparator) + 1, + changedResource.length() - 4 /* .csv */ ); changedDefinitionNames.add(definitionName); } @@ -247,10 +247,9 @@ private List getMainLatestFiles() throws IOException { } private String findAddedTransportVersionName(Set referencedNames, List changedDefinitionNames) throws IOException { - - // First check for unreferenced names. There should only be at most one. If there is more than - // one reference the remaining reference will still be unreferenced when validation runs later - // and the developer will remove the unreferenced name. + // First check for unreferenced names. We only care about the first one. If there is more than one + // validation will fail later and the developer will have to remove one. When that happens, generation + // will re-run and we will fixup the state to use whatever new name remains. for (String referencedName : referencedNames) { Path definitionFile = TransportVersionUtils.definitionFilePath(getResourcesDirectory().get(), referencedName); if (Files.exists(definitionFile) == false) { @@ -258,6 +257,8 @@ private String findAddedTransportVersionName(Set referencedNames, List getMainResources(String subPath) { return Set.of(output.split(System.lineSeparator())); } + private Set getChangedResources(String subPath) { + String output = gitCommand("diff", "--name-only", "main", getResourcesProjectDir().get() + "/" + subPath); + return Set.of(output.split(System.lineSeparator())); + } + // TODO duplicated private TransportVersionLatest readLatestFromMain(String releaseBranch) { return readExistingFile(releaseBranch, this::latestRelativePath, TransportVersionLatest::fromString); From cb9281830477ae90a0436f501be55fa5a8b87cbf Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Fri, 15 Aug 2025 10:03:35 -0700 Subject: [PATCH 13/68] wip --- .../AbstractTransportVersionFuncTest.groovy | 126 ++++++++++++++++++ .../TransportVersionGenerationFuncTest.groovy | 21 +++ ...portVersionManagementPluginFuncTest.groovy | 113 +--------------- .../TransportVersionResourcesPlugin.java | 13 ++ 4 files changed, 161 insertions(+), 112 deletions(-) create mode 100644 build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy create mode 100644 build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy new file mode 100644 index 0000000000000..f49c8f6e53459 --- /dev/null +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport + + +import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest +import org.gradle.testkit.runner.BuildResult +import org.gradle.testkit.runner.TaskOutcome + +class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest { + + /** + * + * @param project + * @param path + * @param content + * @return + */ + def javaResource(String project, String path, String content) { + file("${project}/src/main/resources/${path}").withWriter { writer -> + writer << content + } + } + + def javaSource(String project, String packageName, String className, String imports, String content) { + String packageSlashes = packageName.replace('.', '/') + file("${project}/src/main/java/${packageSlashes}/${className}.java").withWriter { writer -> + writer << """ + package ${packageName}; + ${imports} + public class ${className} { + ${content} + } + """ + } + } + + def definedTransportVersion(String name, String ids) { + javaResource("myserver", "transport/defined/" + name + ".csv", ids) + } + + def definedAndUsedTransportVersion(String name, String ids) { + return definedAndUsedTransportVersion(name, ids, "Test${name.capitalize()}") + } + + def definedAndUsedTransportVersion(String name, String ids, String classname) { + javaSource("myserver", "org.elasticsearch", classname, "", """ + static final TransportVersion usage = TransportVersion.fromName("${name}"); + """) + definedTransportVersion(name, ids) + } + + def latestTransportVersion(String branch, String name, String id) { + javaResource("myserver", "transport/latest/" + branch + ".csv","${name},${id}") + } + + def validateReferencesFails(String project) { + return gradleRunner(":${project}:validateTransportVersionReferences").buildAndFail() + } + + def validateDefinitionsFails() { + return gradleRunner(":myserver:validateTransportVersionDefinitions").buildAndFail() + } + + def assertReferencesFailure(BuildResult result, String project, String expectedOutput) { + result.task(":${project}:validateTransportVersionReferences").outcome == TaskOutcome.FAILED + assertOutputContains(result.output, expectedOutput) + } + + def assertDefinitionsFailure(BuildResult result, String expectedOutput) { + result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.FAILED + assertOutputContains(result.output, expectedOutput) + } + + def setup() { + configurationCacheCompatible = false + internalBuild() + settingsFile << """ + include ':myserver' + include ':myplugin' + """ + file("gradle.properties") << """ + org.elasticsearch.transport.definitionsProject=:myserver + """ + + file("myserver/build.gradle") << """ + apply plugin: 'java-library' + apply plugin: 'elasticsearch.transport-version-references' + apply plugin: 'elasticsearch.transport-version-resources' + """ + definedTransportVersion("existing_91", "8012000") + definedTransportVersion("existing_92", "8123000,8012001") + latestTransportVersion("9.2", "existing_92", "8123000") + latestTransportVersion("9.1", "existing_92", "8012001") + // a mock version of TransportVersion, just here so we can compile Dummy.java et al + javaSource("myserver", "org.elasticsearch", "TransportVersion", "", """ + public static TransportVersion fromName(String name) { + return null; + } + """) + javaSource("myserver", "org.elasticsearch", "Dummy", "", """ + static final TransportVersion existing91 = TransportVersion.fromName("existing_91"); + static final TransportVersion existing92 = TransportVersion.fromName("existing_92"); + """) + + file("myplugin/build.gradle") << """ + apply plugin: 'java-library' + apply plugin: 'elasticsearch.transport-version-references' + + dependencies { + implementation project(":myserver") + } + """ + + setupLocalGitRepo() + execute("git checkout -b main") + execute("git checkout -b test") + } +} diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy new file mode 100644 index 0000000000000..97a3134bc8c5f --- /dev/null +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport + +import org.gradle.testkit.runner.TaskOutcome + +class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTest { + def "test setup works"() { + when: + def result = gradleRunner("generateTransportVersionDefinition").build() + then: + result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + } +} diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPluginFuncTest.groovy index 8eceff86f41fb..6d1a9b02b4872 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPluginFuncTest.groovy @@ -10,120 +10,9 @@ package org.elasticsearch.gradle.internal.transport -import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest -import org.gradle.testkit.runner.BuildResult import org.gradle.testkit.runner.TaskOutcome -class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest { - - /** - * - * @param project - * @param path - * @param content - * @return - */ - def javaResource(String project, String path, String content) { - file("${project}/src/main/resources/${path}").withWriter { writer -> - writer << content - } - } - - def javaSource(String project, String packageName, String className, String imports, String content) { - String packageSlashes = packageName.replace('.', '/') - file("${project}/src/main/java/${packageSlashes}/${className}.java").withWriter { writer -> - writer << """ - package ${packageName}; - ${imports} - public class ${className} { - ${content} - } - """ - } - } - - def definedTransportVersion(String name, String ids) { - javaResource("myserver", "transport/defined/" + name + ".csv", ids) - } - - def definedAndUsedTransportVersion(String name, String ids) { - return definedAndUsedTransportVersion(name, ids, "Test${name.capitalize()}") - } - - def definedAndUsedTransportVersion(String name, String ids, String classname) { - javaSource("myserver", "org.elasticsearch", classname, "", """ - static final TransportVersion usage = TransportVersion.fromName("${name}"); - """) - definedTransportVersion(name, ids) - } - - def latestTransportVersion(String branch, String name, String id) { - javaResource("myserver", "transport/latest/" + branch + ".csv","${name},${id}") - } - - def validateReferencesFails(String project) { - return gradleRunner(":${project}:validateTransportVersionReferences").buildAndFail() - } - - def validateDefinitionsFails() { - return gradleRunner(":myserver:validateTransportVersionDefinitions").buildAndFail() - } - - def assertReferencesFailure(BuildResult result, String project, String expectedOutput) { - result.task(":${project}:validateTransportVersionReferences").outcome == TaskOutcome.FAILED - assertOutputContains(result.output, expectedOutput) - } - - def assertDefinitionsFailure(BuildResult result, String expectedOutput) { - result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.FAILED - assertOutputContains(result.output, expectedOutput) - } - - def setup() { - configurationCacheCompatible = false - internalBuild() - settingsFile << """ - include ':myserver' - include ':myplugin' - """ - file("gradle.properties") << """ - org.elasticsearch.transport.definitionsProject=:myserver - """ - - file("myserver/build.gradle") << """ - apply plugin: 'java-library' - apply plugin: 'elasticsearch.transport-version-references' - apply plugin: 'elasticsearch.transport-version-resources' - """ - definedTransportVersion("existing_91", "8012000") - definedTransportVersion("existing_92", "8123000,8012001") - latestTransportVersion("9.2", "existing_92", "8123000") - latestTransportVersion("9.1", "existing_92", "8012001") - // a mock version of TransportVersion, just here so we can compile Dummy.java et al - javaSource("myserver", "org.elasticsearch", "TransportVersion", "", """ - public static TransportVersion fromName(String name) { - return null; - } - """) - javaSource("myserver", "org.elasticsearch", "Dummy", "", """ - static final TransportVersion existing91 = TransportVersion.fromName("existing_91"); - static final TransportVersion existing92 = TransportVersion.fromName("existing_92"); - """) - - file("myplugin/build.gradle") << """ - apply plugin: 'java-library' - apply plugin: 'elasticsearch.transport-version-references' - - dependencies { - implementation project(":myserver") - } - """ - - setupLocalGitRepo() - execute("git checkout -b main") - execute("git checkout -b test") - } - +class TransportVersionManagementPluginFuncTest extends AbstractTransportVersionFuncTest { def "test setup works"() { when: def result = gradleRunner("validateTransportVersionDefinitions", "validateTransportVersionReferences").build() diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java index d5591bf9a54de..44b6b3aeb544b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java @@ -9,6 +9,8 @@ package org.elasticsearch.gradle.internal.transport; +import org.elasticsearch.gradle.Version; +import org.elasticsearch.gradle.VersionProperties; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; @@ -64,5 +66,16 @@ public void apply(Project project) { project.getTasks().named(JavaPlugin.PROCESS_RESOURCES_TASK_NAME, Copy.class).configure(t -> { t.into("transport/defined", c -> c.from(generateManifestTask)); }); + + var generateDefinitionsTask = project.getTasks() + .register("generateTransportVersionDefinition", GenerateTransportVersionDefinitionTask.class, t -> { + t.setGroup("Transport Versions"); + t.setDescription("(Re)generates a transport version definition file"); + t.getResourcesDirectory().set(getResourcesDirectory(project)); + t.getReferencesFiles().setFrom(tvReferencesConfig); + Version esVersion = VersionProperties.getElasticsearchVersion(); + t.getMainReleaseBranch().set(esVersion.getMajor() + "." + esVersion.getMinor()); + t.getResourcesProjectDir().set(project.getRootProject().getProjectDir().toPath().relativize(project.getProjectDir().toPath()).toString()); + }); } } From 2ecf07ef91eb1b14d95c1246a9ce4115188cb437 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Fri, 15 Aug 2025 11:15:14 -0700 Subject: [PATCH 14/68] iter --- ...enerateTransportVersionDefinitionTask.java | 32 ++++--------------- .../transport/TransportVersionLatest.java | 3 +- .../TransportVersionResourcesPlugin.java | 3 +- 3 files changed, 11 insertions(+), 27 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index 5673b446a0809..cdd883fcf011d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -26,6 +26,7 @@ import org.gradle.process.ExecOperations; import org.gradle.process.ExecResult; +import javax.inject.Inject; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -42,8 +43,6 @@ import java.util.function.Function; import java.util.stream.Collectors; -import javax.inject.Inject; - import static org.elasticsearch.gradle.internal.transport.TransportVersionReference.listFromFile; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.latestFilePath; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.writeDefinitionFile; @@ -240,7 +239,7 @@ private List getMainLatestFiles() throws IOException { List latestFiles = new ArrayList<>(); String latestBasePath = TransportVersionUtils.getResourcePath(getResourcesProjectDir().get(), "latest/"); for (String latestPath : getMainResources(latestBasePath)) { - TransportVersionLatest latest = readExistingFile(latestPath, this::latestRelativePath, TransportVersionLatest::fromString); + TransportVersionLatest latest = readExistingFile(latestPath, Function.identity(), TransportVersionLatest::fromString); latestFiles.add(latest); } return latestFiles; @@ -268,20 +267,18 @@ private String findAddedTransportVersionName(Set referencedNames, List getMainResources(String subPath) { - String output = gitCommand("ls-tree", "--name-only", "-r", "main", getResourcesProjectDir().get() + "/" + subPath); + String output = gitCommand("ls-tree", "--name-only", "-r", "main", subPath); return Set.of(output.split(System.lineSeparator())); } private Set getChangedResources(String subPath) { - String output = gitCommand("diff", "--name-only", "main", getResourcesProjectDir().get() + "/" + subPath); + String output = gitCommand("diff", "--name-only", "main", subPath).strip(); + if (output.isEmpty()) { + return Set.of(); + } return Set.of(output.split(System.lineSeparator())); } - // TODO duplicated - private TransportVersionLatest readLatestFromMain(String releaseBranch) { - return readExistingFile(releaseBranch, this::latestRelativePath, TransportVersionLatest::fromString); - } - // TODO duplicated private T readExistingFile(String name, Function pathFunction, BiFunction parser) { String relativePath = pathFunction.apply(name); @@ -289,21 +286,6 @@ private T readExistingFile(String name, Function pathFunctio return parser.apply(relativePath, content); } - // TODO duplicated - private String latestRelativePath(String branch) { - return relativePath(latestFilePath(getResourcesDirectory().get(), branch)); - } - - // TODO duplicated - private Path resourcesDirPath() { - return getResourcesDirectory().get().getAsFile().toPath(); - } - - // TODO duplicated - private String relativePath(Path file) { - return rootPath.relativize(file).toString(); - } - // TODO duplicated public String gitCommand(String... args) { final ByteArrayOutputStream stdout = new ByteArrayOutputStream(); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionLatest.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionLatest.java index d1f1af543a29f..2d7d9f5e079fd 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionLatest.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionLatest.java @@ -13,7 +13,8 @@ record TransportVersionLatest(String releaseBranch, String name, TransportVersio public static TransportVersionLatest fromString(String filename, String contents) { assert filename.endsWith(".csv"); - String branch = filename.substring(0, filename.length() - 4); + int slashIndex = filename.lastIndexOf('/'); + String branch = filename.substring(slashIndex == -1 ? 0 : (slashIndex + 1), filename.length() - 4); String[] parts = contents.split(","); if (parts.length != 2) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java index eecc9beaa1b42..bf4b1426faded 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java @@ -75,7 +75,8 @@ public void apply(Project project) { t.getReferencesFiles().setFrom(tvReferencesConfig); Version esVersion = VersionProperties.getElasticsearchVersion(); t.getMainReleaseBranch().set(esVersion.getMajor() + "." + esVersion.getMinor()); - t.getResourcesProjectDir().set(project.getRootProject().getProjectDir().toPath().relativize(project.getProjectDir().toPath()).toString()); + t.getResourcesProjectDir() + .set(project.getRootProject().getProjectDir().toPath().relativize(project.getProjectDir().toPath()).toString()); }); } } From 05becfa2c121beb48e3cd2dafe1dd6f65ec1e6e4 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Fri, 15 Aug 2025 11:21:59 -0700 Subject: [PATCH 15/68] spotless --- .../transport/GenerateTransportVersionDefinitionTask.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index cdd883fcf011d..df0371b87f044 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -26,7 +26,6 @@ import org.gradle.process.ExecOperations; import org.gradle.process.ExecResult; -import javax.inject.Inject; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -43,8 +42,9 @@ import java.util.function.Function; import java.util.stream.Collectors; +import javax.inject.Inject; + import static org.elasticsearch.gradle.internal.transport.TransportVersionReference.listFromFile; -import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.latestFilePath; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.writeDefinitionFile; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.writeLatestFile; From 4318792339f8b610d236d5cc75c568403c3cbd3c Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Fri, 15 Aug 2025 15:02:37 -0700 Subject: [PATCH 16/68] Add notes to func test --- .../TransportVersionGenerationFuncTest.groovy | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index 97a3134bc8c5f..821dd4ef0d333 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -18,4 +18,16 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes then: result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS } + + /* + TODO: Add tests that check that: + - TVs added ontop of main in git, but are no longer referenced, are deleted + - name without branches param should fail + - branches without name param should fail (+ other invalid combos) + - multiple branches should create patch versions + - a single branch value should create only a primary id + - a latest file without a corresponding definition file should be reverted to main + - a merge conflict should be resolved, resulting in regeneration of the latest file. + - + */ } From 367e52f8afb1e3d26297216edbbe68de364f5e1b Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Mon, 18 Aug 2025 15:36:43 -0700 Subject: [PATCH 17/68] iter --- .../TransportVersionGenerationFuncTest.groovy | 38 +++++++++++++++++++ ...enerateTransportVersionDefinitionTask.java | 2 +- .../transport/TransportVersionReference.java | 2 +- .../TransportVersionReferencesPlugin.java | 2 +- 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index 821dd4ef0d333..8abae5a79163b 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -9,8 +9,12 @@ package org.elasticsearch.gradle.internal.transport +import org.elasticsearch.gradle.util.Pair +import org.gradle.internal.impldep.com.google.common.collect.Streams import org.gradle.testkit.runner.TaskOutcome +import java.util.stream.Stream + class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTest { def "test setup works"() { when: @@ -19,6 +23,40 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS } + def "A definition should be generated for an undefined reference"() { + given: + String tvName = "potato_tv" + when: + def result = gradleRunner("generateTransportVersionDefinition", "--name=" + tvName, "--increment=1").build() + then: + result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + validateDefinitionFile(tvName, List.of("9.2")) + } + + void validateDefinitionFile(String name, List branches) { // TODO add primary increment + String filename = "myserver/src/main/resources/transport/definitions/named/" + name + ".csv" + assert file(filename).exists() + + String contents = file(filename).text + assert contents.strip().isEmpty() == false + String[] x = contents.strip().split(",") + Stream ids = Arrays.stream(x).map(Integer::valueOf) + assert branches.size() == ids.count(): "The definition file does not have the correct number of ids" + + def latestInfo = branches.stream() + .map { file("myserver/src/main/resources/transport/latest/${it}.csv").text } + .map { contents.strip().split(",") } + .map { Pair.of(it.first(), Integer.valueOf(it.last())) } + + Streams.zip(ids, latestInfo, { (id, latest) -> Pair.of(id, latest) }) + .forEach { + (id, Pair latest) -> { + assert name == latest.left(): "The latest file should contain the same name as the new transport version" + assert id == latest.right(): "The latest file should contain the same id as the new transport version" + } + } + } + /* TODO: Add tests that check that: - TVs added ontop of main in git, but are no longer referenced, are deleted diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index df0371b87f044..f1ef87162662b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -216,7 +216,7 @@ private List getChangedDefinitionNames() throws IOException { String namedDefinitionsBasePath = TransportVersionUtils.getResourcePath(getResourcesProjectDir().get(), "definitions/named/"); for (String changedResource : getChangedResources(namedDefinitionsBasePath)) { String definitionName = changedResource.substring( - changedResource.lastIndexOf(File.pathSeparator) + 1, + changedResource.lastIndexOf(File.separator) + 1, changedResource.length() - 4 /* .csv */ ); changedDefinitionNames.add(definitionName); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionReference.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionReference.java index 5c89b41db799d..cc67bd8286891 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionReference.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionReference.java @@ -26,7 +26,7 @@ record TransportVersionReference(String name, String location) { private static final Attribute REFERENCES_ATTRIBUTE = Attribute.of("transport-version-references", Boolean.class); static List listFromFile(Path file) throws IOException { - assert file.endsWith(".csv"); + assert file.toString().endsWith(".csv") : file + " does not end in .csv"; List results = new ArrayList<>(); for (String line : Files.readAllLines(file, StandardCharsets.UTF_8)) { String[] parts = line.split(",", 2); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionReferencesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionReferencesPlugin.java index 60012feac5da3..dd3541ba6147b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionReferencesPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionReferencesPlugin.java @@ -32,7 +32,7 @@ public void apply(Project project) { t.setDescription("Collects all TransportVersion references used throughout the project"); SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); t.getClassPath().setFrom(mainSourceSet.getOutput()); - t.getOutputFile().set(project.getLayout().getBuildDirectory().file("transport-version/references.txt")); + t.getOutputFile().set(project.getLayout().getBuildDirectory().file("transport-version/references.csv")); }); Configuration tvReferencesConfig = project.getConfigurations().create("transportVersionReferences", c -> { From 50cde19d35cd64f1ecfbbca9ea3ed8146788384f Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Tue, 19 Aug 2025 10:32:59 -0700 Subject: [PATCH 18/68] iter --- .../TransportVersionGenerationFuncTest.groovy | 51 +++++++++++-------- ...enerateTransportVersionDefinitionTask.java | 2 +- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index 8abae5a79163b..b38906e66ae49 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -9,12 +9,8 @@ package org.elasticsearch.gradle.internal.transport -import org.elasticsearch.gradle.util.Pair -import org.gradle.internal.impldep.com.google.common.collect.Streams import org.gradle.testkit.runner.TaskOutcome -import java.util.stream.Stream - class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTest { def "test setup works"() { when: @@ -27,34 +23,45 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes given: String tvName = "potato_tv" when: - def result = gradleRunner("generateTransportVersionDefinition", "--name=" + tvName, "--increment=1").build() + def result = gradleRunner( + "generateTransportVersionDefinition", + "--name=" + tvName, + "--increment=1", + "--branches=main" + ).build() then: result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS validateDefinitionFile(tvName, List.of("9.2")) } - void validateDefinitionFile(String name, List branches) { // TODO add primary increment - String filename = "myserver/src/main/resources/transport/definitions/named/" + name + ".csv" + def "Missing required arguments should fail, state should remain unaltered"() { + // TODO + } + + // TODO add a check for the increment, primary and patch + void validateDefinitionFile(String definitionName, List branches) { + String filename = "myserver/src/main/resources/transport/definitions/named/" + definitionName + ".csv" assert file(filename).exists() - String contents = file(filename).text - assert contents.strip().isEmpty() == false - String[] x = contents.strip().split(",") - Stream ids = Arrays.stream(x).map(Integer::valueOf) - assert branches.size() == ids.count(): "The definition file does not have the correct number of ids" + String definitionFileText = file(filename).text.strip() + assert definitionFileText.isEmpty() == false: "The definition file must not be empty" + List definitionIDs = Arrays.stream(definitionFileText.split(",")).map(Integer::valueOf).toList() + assert branches.size() == definitionIDs.size(): "The definition file does not have an id for each branch" - def latestInfo = branches.stream() + def latestNamesToIds = branches.stream() .map { file("myserver/src/main/resources/transport/latest/${it}.csv").text } - .map { contents.strip().split(",") } - .map { Pair.of(it.first(), Integer.valueOf(it.last())) } + .map { it.strip().split(",") } + .map { new Tuple2(it.first(), Integer.valueOf(it.last())) } + .toList() + + for (int i = 0; i < definitionIDs.size(); i++) { + int definitionID = definitionIDs[i] + String nameInLatest = latestNamesToIds[i].getV1() + def idInLatest = latestNamesToIds[i].getV2() - Streams.zip(ids, latestInfo, { (id, latest) -> Pair.of(id, latest) }) - .forEach { - (id, Pair latest) -> { - assert name == latest.left(): "The latest file should contain the same name as the new transport version" - assert id == latest.right(): "The latest file should contain the same id as the new transport version" - } - } + assert definitionName.equals(nameInLatest): "The latest and definition names must match" + assert definitionID == idInLatest: "The latest and definition ids must match" + } } /* diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index f1ef87162662b..59199b0f65964 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -165,7 +165,7 @@ private List updateLatestFiles(Path resourcesDir, String nam } private Set getTargetReleaseBranches() { - if (getBranches().isPresent()) { + if (getBranches().get().isEmpty() == false) { return getBranches().get() .stream() .map(branch -> branch.equals("main") ? getMainReleaseBranch().get() : branch) From 0c5b6e88f9ae2a55fce985793c195173c3b970aa Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Tue, 19 Aug 2025 15:01:27 -0700 Subject: [PATCH 19/68] iter --- .../AbstractTransportVersionFuncTest.groovy | 10 +- .../TransportVersionGenerationFuncTest.groovy | 123 ++++++++++++++---- ...enerateTransportVersionDefinitionTask.java | 10 +- .../transport/TransportVersionId.java | 2 +- .../TransportVersionResourcesPlugin.java | 3 + .../transport/TransportVersionUtils.java | 2 +- ...ValidateTransportVersionResourcesTask.java | 2 +- 7 files changed, 121 insertions(+), 31 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy index 5abf915942885..678f34551160c 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy @@ -55,10 +55,18 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest { } def definedAndUsedTransportVersion(String name, String ids, String classname) { + referencedTransportVersion(name, classname); + namedTransportVersion(name, ids) + } + + def referencedTransportVersion(String name, String classname) { javaSource("myserver", "org.elasticsearch", classname, "", """ static final TransportVersion usage = TransportVersion.fromName("${name}"); """) - namedTransportVersion(name, ids) + } + + def referencedTransportVersion(String name) { + return referencedTransportVersion(name, "Test${name.capitalize()}") } def latestTransportVersion(String branch, String name, String id) { diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index b38906e66ae49..a615a15e2cc01 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -19,41 +19,105 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS } - def "A definition should be generated for an undefined reference"() { + def "A definition should be generated for the given branches"(List branches) { given: - String tvName = "potato_tv" + String tvName = "test_tv_patch_ids" + referencedTransportVersion(tvName) + List latestBranchesToOriginalIds = readLatestFiles(branches) + when: def result = gradleRunner( + ":myserver:validateTransportVersionDefinitions", "generateTransportVersionDefinition", "--name=" + tvName, - "--increment=1", - "--branches=main" + "--branches=" + branches.join(",") + ).build() + + then: + result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + validateDefinitionFile(tvName, latestBranchesToOriginalIds) + + where: + branches << [ + ["9.2"], + ["9.2", "9.1"] + ] + } + + def "A definition should be generated when the name isn't specified"() { + given: + String tvName = "test_tv_patch_ids" + referencedTransportVersion(tvName) + List branches = ["9.2"] + List latestBranchesToOriginalIds = readLatestFiles(branches) + + when: + def result = gradleRunner( + ":myserver:validateTransportVersionDefinitions", + "generateTransportVersionDefinition", + "--branches=" + branches.join(",") ).build() + then: result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS - validateDefinitionFile(tvName, List.of("9.2")) + result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + validateDefinitionFile(tvName, latestBranchesToOriginalIds) + } + + def "Should fail if branches are omitted and state should remain unaltered"() { + when: + def generateResult = gradleRunner("generateTransportVersionDefinition", "--name=no_branches").buildAndFail() + def validateResult = gradleRunner("validateTransportVersionDefinitions").build() + + then: + generateResult.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.FAILED + validateResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + } + + + /* + TODO: Add tests that check that: + - TVs added ontop of main in git, but are no longer referenced, are deleted + - name without branches param should fail + - a latest file without a corresponding definition file should be reverted to main + - a merge conflict should be resolved, resulting in regeneration of the latest file. + - + */ + + List readLatestFiles(List branches) { + return branches.stream() + .map { readLatestFile(it) } + .toList() } - def "Missing required arguments should fail, state should remain unaltered"() { - // TODO + LatestFile readLatestFile(String branch) { + String latestFileText = file( + "myserver/src/main/resources/transport/latest/${branch}.csv" + ).text.strip() + assert latestFileText.isEmpty() == false: "The latest file must not be empty" + List parts = latestFileText.split(",") + assert parts.size() == 2: "The latest file must contain exactly two parts" + return new LatestFile(branch, parts[0], Integer.valueOf(parts[1])) } - // TODO add a check for the increment, primary and patch - void validateDefinitionFile(String definitionName, List branches) { + void validateDefinitionFile(String definitionName, List originalLatestFiles, Integer primaryIncrement = 1000) { String filename = "myserver/src/main/resources/transport/definitions/named/" + definitionName + ".csv" assert file(filename).exists() String definitionFileText = file(filename).text.strip() assert definitionFileText.isEmpty() == false: "The definition file must not be empty" List definitionIDs = Arrays.stream(definitionFileText.split(",")).map(Integer::valueOf).toList() - assert branches.size() == definitionIDs.size(): "The definition file does not have an id for each branch" + assert originalLatestFiles.size() == definitionIDs.size(): "The definition file does not have an id for each latest file" - def latestNamesToIds = branches.stream() + def latestNamesToIds = originalLatestFiles.stream() + .map { it.branch } .map { file("myserver/src/main/resources/transport/latest/${it}.csv").text } .map { it.strip().split(",") } .map { new Tuple2(it.first(), Integer.valueOf(it.last())) } .toList() + for (int i = 0; i < definitionIDs.size(); i++) { int definitionID = definitionIDs[i] String nameInLatest = latestNamesToIds[i].getV1() @@ -61,18 +125,33 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assert definitionName.equals(nameInLatest): "The latest and definition names must match" assert definitionID == idInLatest: "The latest and definition ids must match" + + int originalID = originalLatestFiles[i].id + if (i == 0) { + assert definitionID == originalID + primaryIncrement: + "The primary version ID should be incremented by ${primaryIncrement} from the main branch latest file" + } else { + assert definitionID == originalID + 1: + "The patch version ID should be incremented by 1 from the primary version latest file" + } + } } - /* - TODO: Add tests that check that: - - TVs added ontop of main in git, but are no longer referenced, are deleted - - name without branches param should fail - - branches without name param should fail (+ other invalid combos) - - multiple branches should create patch versions - - a single branch value should create only a primary id - - a latest file without a corresponding definition file should be reverted to main - - a merge conflict should be resolved, resulting in regeneration of the latest file. - - - */ + class LatestFile { + String branch + String name + Integer id + + LatestFile(String branch, String name, Integer id) { + this.branch = branch + this.name = name + this.id = id + } + + @Override + String toString() { + return "LatestFile(branch=${branch}, name=${name}, id=${id})" + } + } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index 59199b0f65964..0e174756a8063 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -13,7 +13,6 @@ import org.gradle.api.Project; import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.DirectoryProperty; -import org.gradle.api.provider.ListProperty; import org.gradle.api.provider.Property; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputDirectory; @@ -33,6 +32,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -86,11 +86,11 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask @Optional // In CI we find these from the github PR labels @Input @Option(option = "branches", description = "The branches for which to generate IDs, e.g. --branches=\"main,9.1\"") - public abstract ListProperty getBranches(); + public abstract Property getBranches(); @Input @Optional - @Option(option = "increment", description = "TBD") + @Option(option = "increment", description = "The amount to increment the primary id for the main branch") public abstract Property getPrimaryIncrement(); @Input @@ -161,13 +161,13 @@ private List updateLatestFiles(Path resourcesDir, String nam } } + Collections.sort(ids); return ids; } private Set getTargetReleaseBranches() { if (getBranches().get().isEmpty() == false) { - return getBranches().get() - .stream() + return Arrays.stream(getBranches().get().split(",")) .map(branch -> branch.equals("main") ? getMainReleaseBranch().get() : branch) .collect(Collectors.toSet()); } else { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionId.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionId.java index 32d91b549850c..29c8f042b249b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionId.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionId.java @@ -62,7 +62,7 @@ public TransportVersionId bumpComponent(Component component) { @Override public int compareTo(TransportVersionId o) { - return Integer.compare(complete, o.complete); + return Integer.compare(o.complete, complete); } @Override diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java index bf4b1426faded..6b963ad7d9d57 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java @@ -73,10 +73,13 @@ public void apply(Project project) { t.setDescription("(Re)generates a transport version definition file"); t.getResourcesDirectory().set(getResourcesDirectory(project)); t.getReferencesFiles().setFrom(tvReferencesConfig); + t.getPrimaryIncrement().convention(1000); Version esVersion = VersionProperties.getElasticsearchVersion(); t.getMainReleaseBranch().set(esVersion.getMajor() + "." + esVersion.getMinor()); t.getResourcesProjectDir() .set(project.getRootProject().getProjectDir().toPath().relativize(project.getProjectDir().toPath()).toString()); }); + + validateTask.configure(t -> t.mustRunAfter(generateDefinitionsTask)); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 772ee135bb4e1..7f3614fc8d99d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -36,7 +36,7 @@ static TransportVersionDefinition readDefinitionFile(Path file) throws IOExcepti static void writeDefinitionFile(Path resourcesDir, TransportVersionDefinition definition) throws IOException { Files.writeString( resourcesDir.resolve("definitions/named").resolve(definition.name() + ".csv"), - definition.ids().stream().map(id -> String.valueOf(id.complete())).collect(Collectors.joining(",")) + "\n", + definition.ids().stream().map(Object::toString).collect(Collectors.joining(",")) + "\n", StandardCharsets.UTF_8 ); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java index 03873bfc20b25..6d4aa5de3172f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java @@ -197,7 +197,7 @@ private void recordAndValidateDefinition(TransportVersionDefinition definition) if (definition.ids().isEmpty()) { throwDefinitionFailure(definition.name(), "does not contain any ids"); } - if (Comparators.isInOrder(definition.ids(), Comparator.reverseOrder()) == false) { + if (Comparators.isInOrder(definition.ids(), Comparator.naturalOrder()) == false) { throwDefinitionFailure(definition.name(), "does not have ordered ids"); } for (int ndx = 0; ndx < definition.ids().size(); ++ndx) { From 1cfe88c053841abc924a6621b7e81d99b4ff36a8 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 20 Aug 2025 13:53:53 +0000 Subject: [PATCH 20/68] [CI] Auto commit changes from spotless --- .../GenerateTransportVersionDefinitionTask.java | 12 ++++++++++-- .../transport/TransportVersionResourcesService.java | 12 +++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index 6864fcde1d6da..a14b750bb45ec 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -192,7 +192,11 @@ private void resetAllLatestFiles(TransportVersionResourcesService resources) thr } } - private void removeUnusedNamedDefinitions(TransportVersionResourcesService resources, Set referencedNames, List changedDefinitions) throws IOException { + private void removeUnusedNamedDefinitions( + TransportVersionResourcesService resources, + Set referencedNames, + List changedDefinitions + ) throws IOException { for (TransportVersionDefinition definition : changedDefinitions) { if (referencedNames.contains(definition.name()) == false) { // we added this definition file, but it's now unreferenced, so delete it @@ -202,7 +206,11 @@ private void removeUnusedNamedDefinitions(TransportVersionResourcesService resou } } - private String findAddedTransportVersionName(TransportVersionResourcesService resources, Set referencedNames, List changedDefinitions) throws IOException { + private String findAddedTransportVersionName( + TransportVersionResourcesService resources, + Set referencedNames, + List changedDefinitions + ) throws IOException { // First check for unreferenced names. We only care about the first one. If there is more than one // validation will fail later and the developer will have to remove one. When that happens, generation // will re-run and we will fixup the state to use whatever new name remains. diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java index ce111067327f9..6952e512f50bb 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java @@ -15,7 +15,6 @@ import org.gradle.process.ExecOperations; import org.gradle.process.ExecResult; -import javax.inject.Inject; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -32,6 +31,8 @@ import java.util.function.BiFunction; import java.util.stream.Collectors; +import javax.inject.Inject; + /** * An encapsulation of operations on transport version resources. * @@ -147,13 +148,14 @@ Path getRepositoryPath(TransportVersionDefinition definition) { void writeNamedDefinition(TransportVersionDefinition definition) throws IOException { Path path = transportResourcesDir.resolve(getNamedDefinitionRelativePath(definition.name())); - Files.writeString(path, - definition.ids().stream().map(Object::toString).collect(Collectors.joining(",")) + "\n", - StandardCharsets.UTF_8 + Files.writeString( + path, + definition.ids().stream().map(Object::toString).collect(Collectors.joining(",")) + "\n", + StandardCharsets.UTF_8 ); } - void deleteNamedDefinition(String name) throws IOException{ + void deleteNamedDefinition(String name) throws IOException { Path path = transportResourcesDir.resolve(getNamedDefinitionRelativePath(name)); Files.deleteIfExists(path); } From 8d13cec7fce641b3a1e75b812ac6a4da2eb86ab6 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Wed, 20 Aug 2025 14:48:13 -0700 Subject: [PATCH 21/68] iter --- .../AbstractTransportVersionFuncTest.groovy | 11 ++ .../TransportVersionGenerationFuncTest.groovy | 111 ++++++++++++++++-- .../TransportVersionResourcesService.java | 4 +- 3 files changed, 114 insertions(+), 12 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy index 688e3de555281..4b85ce0a8ee99 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy @@ -34,6 +34,13 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest { } } + def deleteJavaSource(String project, String packageName, String className) { + String packageSlashes = packageName.replace('.', '/') + def filePath = "${project}/src/main/java/${packageSlashes}/${className}.java" + assert file(filePath).exists(): "File does not exist: ${filePath}" + file(filePath).delete() + } + def namedTransportVersion(String name, String ids) { javaResource("myserver", "transport/definitions/named/" + name + ".csv", ids) } @@ -61,6 +68,10 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest { return referencedTransportVersion(name, "Test${name.capitalize()}") } + def deleteTransportVersionReference(String name) { + deleteJavaSource("myserver", "org.elasticsearch", "Test${name.capitalize()}") + } + def latestTransportVersion(String branch, String name, String id) { javaResource("myserver", "transport/latest/" + branch + ".csv","${name},${id}") } diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index a615a15e2cc01..09758ac2b5637 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -14,30 +14,39 @@ import org.gradle.testkit.runner.TaskOutcome class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTest { def "test setup works"() { when: - def result = gradleRunner("generateTransportVersionDefinition").build() + def result = gradleRunner( + ":myserver:generateTransportVersionDefinition", + ":myserver:validateTransportVersionDefinitions" + ).build() + then: result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS } - def "A definition should be generated for the given branches"(List branches) { + def "A definition should be generated when specified by an arg but no code reference exists"(List branches) { given: String tvName = "test_tv_patch_ids" - referencedTransportVersion(tvName) List latestBranchesToOriginalIds = readLatestFiles(branches) - when: + when: "generation is run with a name specified and no code references" def result = gradleRunner( - ":myserver:validateTransportVersionDefinitions", "generateTransportVersionDefinition", "--name=" + tvName, "--branches=" + branches.join(",") ).build() - then: + then: "The generation task should succeed and create the definition file" result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS - result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS validateDefinitionFile(tvName, latestBranchesToOriginalIds) + when: "A reference is added" + referencedTransportVersion(tvName) + def validateResult = gradleRunner("validateTransportVersionDefinitions").build() + + then: "The full validation should succeed now that the reference exists" + validateResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + where: branches << [ ["9.2"], @@ -45,7 +54,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes ] } - def "A definition should be generated when the name isn't specified"() { + def "A definition should be generated when the name arg isn't specified but a code reference exists"() { given: String tvName = "test_tv_patch_ids" referencedTransportVersion(tvName) @@ -65,7 +74,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes validateDefinitionFile(tvName, latestBranchesToOriginalIds) } - def "Should fail if branches are omitted and state should remain unaltered"() { + def "Generation should fail if the branches arg is omitted, the state should remain unaltered"() { when: def generateResult = gradleRunner("generateTransportVersionDefinition", "--name=no_branches").buildAndFail() def validateResult = gradleRunner("validateTransportVersionDefinitions").build() @@ -75,11 +84,91 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes validateResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS } + def "Latest file modifications should be reverted"( + List branchesParam, + List latestFilesModified, + String name + ) { + given: + def originalLatestFiles = latestFilesModified.stream().map { readLatestFile(it) }.toList() + + when: "The latest files are modified then generation is run without a name" + originalLatestFiles.forEach { + latestTransportVersion(it.branch, it.name + "_modification", (it.id + 1).toString()) + } + def args = [ + ":myserver:validateTransportVersionDefinitions", + ":myserver:generateTransportVersionDefinition" + ] + if (branchesParam != null) { + args.add("--branches=" + branchesParam.join(",")) + } + if (name != null) { + referencedTransportVersion(name) + } + def result = gradleRunner(args.toArray(new String[0])).build() + + then: "The generation and validation tasks should succeed, and the latest files should be reverted" + result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + originalLatestFiles.forEach { originalLatest -> + def latest = readLatestFile(originalLatest.branch) + assert latest.branch == originalLatest.branch + assert latest.id == originalLatest.id + } + + where: + branchesParam | latestFilesModified | name + null | ["9.2"] | null + null | ["9.2", "9.1"] | null + ["9.2", "9.1"] | ["9.2"] | null + ["9.2"] | ["9.1"] | "test_tv" // TODO legitimate bug? + } + + // TODO this test is finding a legitimate bug + def "definitions that are no longer referenced should be deleted"(List branches) { + given: + String definitionName = "test_tv_patch_ids" + referencedTransportVersion(definitionName) + List originalLatestFiles = readLatestFiles(branches) + + when: "The definition is generated" + def result = gradleRunner( + ":myserver:validateTransportVersionDefinitions", + ":myserver:generateTransportVersionDefinition", + "--name=" + definitionName, + "--branches=" + branches.join(",") + ).build() + + then: "The generation task should succeed and create the definition file" + result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + validateDefinitionFile(definitionName, originalLatestFiles) + + when: "The reference is removed and the generation is run again" + deleteTransportVersionReference(definitionName) + def secondResult = gradleRunner( + ":myserver:validateTransportVersionDefinitions", + ":myserver:generateTransportVersionDefinition", + "--branches=" + branches.join(",") + ).build() + + then: "The generation task should succeed and the definition file should be deleted" + !file("myserver/src/main/resources/transport/definitions/named/${definitionName}.csv").exists() + secondResult.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + secondResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + + where: + branches << [ + ["9.2"], + ["9.2", "9.1"] + ] + } + /* TODO: Add tests that check that: - - TVs added ontop of main in git, but are no longer referenced, are deleted - - name without branches param should fail + - - a latest file without a corresponding definition file should be reverted to main - a merge conflict should be resolved, resulting in regeneration of the latest file. - diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java index 6952e512f50bb..a87484ec72a72 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java @@ -127,9 +127,10 @@ List getChangedNamedDefinitions() throws IOException List changedDefinitions = new ArrayList<>(); String namedPrefix = NAMED_DIR.toString(); for (String changedPath : getChangedResources()) { - if (changedPath.startsWith(namedPrefix) == false) { + if (changedPath.contains(namedPrefix) == false) { // TODO make this more robust continue; } + // TODO why are we getting the main file here? Shouldn't we just read the changed file directly? TransportVersionDefinition definition = getMainFile(changedPath, TransportVersionDefinition::fromString); changedDefinitions.add(definition); } @@ -224,6 +225,7 @@ private Set getMainResources() { private Set getChangedResources() { if (changedResources.get() == null) { synchronized (changedResources) { + // gitCommand("add", "."); // TODO this finds the files that have been added without being committed. String output = gitCommand("diff", "--name-only", "main", "."); HashSet resources = new HashSet<>(); From f43dd4bcd80f594bb66d357cfc728d68dc9b85fe Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 20 Aug 2025 17:12:38 -0700 Subject: [PATCH 22/68] fix changed files helper to also get untracked files --- .../TransportVersionGenerationFuncTest.groovy | 19 +++++++++++++++ ...enerateTransportVersionDefinitionTask.java | 20 ++++++++-------- .../TransportVersionResourcesService.java | 24 ++++++++++++------- 3 files changed, 44 insertions(+), 19 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index 09758ac2b5637..db51cac29d6cd 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -165,6 +165,25 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes ] } + def "unreferenced definitions should be deleted"() { + given: + namedTransportVersion("old_name", "8124000") + referencedTransportVersion("new_name") + + when: + def result = gradleRunner( + ":myserver:validateTransportVersionDefinitions", + ":myserver:generateTransportVersionDefinition", + "--name=new_name" , + "--branches=9.2" + ).build() + + then: + !file("myserver/src/main/resources/transport/definitions/named/old_name.csv").exists() + result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + } + /* TODO: Add tests that check that: diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index a14b750bb45ec..4b959946df8ba 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -100,10 +100,10 @@ public GenerateTransportVersionDefinitionTask(ExecOperations execOperations) { public void run() throws IOException { TransportVersionResourcesService resources = getResources().get(); Set referencedNames = TransportVersionReference.collectNames(getReferencesFiles()); - List changedDefinitions = resources.getChangedNamedDefinitions(); + List changedDefinitionNames = resources.getChangedNamedDefinitionNames(); String name = getTransportVersionName().isPresent() ? getTransportVersionName().get() - : findAddedTransportVersionName(resources, referencedNames, changedDefinitions); + : findAddedTransportVersionName(resources, referencedNames, changedDefinitionNames); if (name.isEmpty()) { resetAllLatestFiles(resources); @@ -113,7 +113,7 @@ public void run() throws IOException { resources.writeNamedDefinition(new TransportVersionDefinition(name, ids)); } - removeUnusedNamedDefinitions(resources, referencedNames, changedDefinitions); + removeUnusedNamedDefinitions(resources, referencedNames, changedDefinitionNames); } private List updateLatestFiles(TransportVersionResourcesService resources, String name) throws IOException { @@ -195,13 +195,13 @@ private void resetAllLatestFiles(TransportVersionResourcesService resources) thr private void removeUnusedNamedDefinitions( TransportVersionResourcesService resources, Set referencedNames, - List changedDefinitions + List changedDefinitions ) throws IOException { - for (TransportVersionDefinition definition : changedDefinitions) { - if (referencedNames.contains(definition.name()) == false) { + for (String definitionName : changedDefinitions) { + if (referencedNames.contains(definitionName) == false) { // we added this definition file, but it's now unreferenced, so delete it - getLogger().lifecycle("Deleting unreferenced named transport version definition [" + definition.name() + "]"); - resources.deleteNamedDefinition(definition.name()); + getLogger().lifecycle("Deleting unreferenced named transport version definition [" + definitionName + "]"); + resources.deleteNamedDefinition(definitionName); } } } @@ -209,7 +209,7 @@ private void removeUnusedNamedDefinitions( private String findAddedTransportVersionName( TransportVersionResourcesService resources, Set referencedNames, - List changedDefinitions + List changedDefinitions ) throws IOException { // First check for unreferenced names. We only care about the first one. If there is more than one // validation will fail later and the developer will have to remove one. When that happens, generation @@ -225,7 +225,7 @@ private String findAddedTransportVersionName( if (changedDefinitions.isEmpty()) { return ""; } else { - return changedDefinitions.getFirst().name(); + return changedDefinitions.getFirst(); } } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java index a87484ec72a72..689a46027a3ec 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java @@ -123,16 +123,15 @@ TransportVersionDefinition getNamedDefinitionFromMain(String name) { return getMainFile(resourcePath, TransportVersionDefinition::fromString); } - List getChangedNamedDefinitions() throws IOException { - List changedDefinitions = new ArrayList<>(); + List getChangedNamedDefinitionNames() { + List changedDefinitions = new ArrayList<>(); String namedPrefix = NAMED_DIR.toString(); for (String changedPath : getChangedResources()) { - if (changedPath.contains(namedPrefix) == false) { // TODO make this more robust + if (changedPath.contains(namedPrefix) == false) { continue; } - // TODO why are we getting the main file here? Shouldn't we just read the changed file directly? - TransportVersionDefinition definition = getMainFile(changedPath, TransportVersionDefinition::fromString); - changedDefinitions.add(definition); + String name = changedPath.substring(namedPrefix.length() + 1 /* skip slash */, changedPath.length() - 4 /* .csv */); + changedDefinitions.add(name); } return changedDefinitions; } @@ -225,11 +224,18 @@ private Set getMainResources() { private Set getChangedResources() { if (changedResources.get() == null) { synchronized (changedResources) { + HashSet resources = new HashSet<>(); + // gitCommand("add", "."); // TODO this finds the files that have been added without being committed. - String output = gitCommand("diff", "--name-only", "main", "."); + String diffOutput = gitCommand("diff", "--name-only", "main", "."); + if (diffOutput.strip().isEmpty() == false) { + Collections.addAll(resources, diffOutput.split(System.lineSeparator())); + } - HashSet resources = new HashSet<>(); - Collections.addAll(resources, output.split(System.lineSeparator())); + String untrackedOutput = gitCommand("ls-files", "--others", "--exclude-standard"); + if (untrackedOutput.strip().isEmpty() == false) { + Collections.addAll(resources, untrackedOutput.split(System.lineSeparator())); + } changedResources.set(resources); } } From 975a87ef932cb24d9872c06078e38d743f80a714 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Thu, 21 Aug 2025 07:45:21 -0700 Subject: [PATCH 23/68] iter --- .../TransportVersionGenerationFuncTest.groovy | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index db51cac29d6cd..8ed884cc57ff2 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -84,7 +84,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes validateResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS } - def "Latest file modifications should be reverted"( + def "Latest file modifications should be reverted to their original state on main"( List branchesParam, List latestFilesModified, String name @@ -92,7 +92,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes given: def originalLatestFiles = latestFilesModified.stream().map { readLatestFile(it) }.toList() - when: "The latest files are modified then generation is run without a name" + when: "generation should revert the latest files to their original state" originalLatestFiles.forEach { latestTransportVersion(it.branch, it.name + "_modification", (it.id + 1).toString()) } @@ -116,17 +116,18 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assert latest.branch == originalLatest.branch assert latest.id == originalLatest.id } + // TODO assert that the definition file is created if a name is specified, and contains the correct incremented IDs where: branchesParam | latestFilesModified | name - null | ["9.2"] | null - null | ["9.2", "9.1"] | null - ["9.2", "9.1"] | ["9.2"] | null +// null | ["9.2"] | null +// null | ["9.2", "9.1"] | null +// ["9.2", "9.1"] | ["9.2"] | null ["9.2"] | ["9.1"] | "test_tv" // TODO legitimate bug? } // TODO this test is finding a legitimate bug - def "definitions that are no longer referenced should be deleted"(List branches) { + def "definitions that are no longer referenced should be deleted and latest files reverted"(List branches) { given: String definitionName = "test_tv_patch_ids" referencedTransportVersion(definitionName) @@ -165,7 +166,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes ] } - def "unreferenced definitions should be deleted"() { + def "unreferenced definition files should be deleted"() { given: namedTransportVersion("old_name", "8124000") referencedTransportVersion("new_name") @@ -190,6 +191,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes - - a latest file without a corresponding definition file should be reverted to main - a merge conflict should be resolved, resulting in regeneration of the latest file. + - add a reference, generate, remove the reference, add another reference, generate - */ From 2e60a3bb57dca826c88624eafac19f8fd1d67faf Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Thu, 21 Aug 2025 09:39:02 -0700 Subject: [PATCH 24/68] iter --- .../TransportVersionGenerationFuncTest.groovy | 52 ++++++++++++------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index 8ed884cc57ff2..6571a31d15d2b 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -90,12 +90,15 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes String name ) { given: - def originalLatestFiles = latestFilesModified.stream().map { readLatestFile(it) }.toList() + List originalModifiedLatestFiles = readLatestFiles(latestFilesModified) + List originalBranchesLatestFiles = readLatestFiles(branchesParam) - when: "generation should revert the latest files to their original state" - originalLatestFiles.forEach { + when: "We modify the latest files" + originalModifiedLatestFiles.forEach { latestTransportVersion(it.branch, it.name + "_modification", (it.id + 1).toString()) } + + and: "We run the generation task" def args = [ ":myserver:validateTransportVersionDefinitions", ":myserver:generateTransportVersionDefinition" @@ -104,26 +107,39 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes args.add("--branches=" + branchesParam.join(",")) } if (name != null) { - referencedTransportVersion(name) + referencedTransportVersion(name) // Ensure the definition file is referenced to allow validation to succeed } def result = gradleRunner(args.toArray(new String[0])).build() - then: "The generation and validation tasks should succeed, and the latest files should be reverted" + then: "The generation and validation tasks should succeed" result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS - originalLatestFiles.forEach { originalLatest -> - def latest = readLatestFile(originalLatest.branch) - assert latest.branch == originalLatest.branch - assert latest.id == originalLatest.id + + and: "The modified latest files should be reverted if there is no name specified or they are not specified in the branches param" + originalModifiedLatestFiles.forEach { originalLatest -> + boolean noNameSpecified = name == null + boolean modifiedNotInBranchesParam = branchesParam != null && branchesParam.contains(originalLatest.branch) == false + if (noNameSpecified || modifiedNotInBranchesParam) { + def latest = readLatestFile(originalLatest.branch) + assert latest.branch == originalLatest.branch + assert latest.id == originalLatest.id + } + } + + and: "The latest files for the branches param should be incremented correctly" + if (name != null) { + validateDefinitionFile(name, originalBranchesLatestFiles,) } - // TODO assert that the definition file is created if a name is specified, and contains the correct incremented IDs where: branchesParam | latestFilesModified | name -// null | ["9.2"] | null -// null | ["9.2", "9.1"] | null -// ["9.2", "9.1"] | ["9.2"] | null - ["9.2"] | ["9.1"] | "test_tv" // TODO legitimate bug? + null | ["9.2"] | null + null | ["9.2", "9.1"] | null + ["9.2", "9.1"] | ["9.2"] | null + ["9.2"] | ["9.1"] | null + ["9.2"] | ["9.1"] | "test_tv" // TODO legitimate bug, need to always clean up latest. + ["9.2", "9.1"] | ["9.2", "9.1"] | "test_tv" + } // TODO this test is finding a legitimate bug @@ -173,10 +189,10 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes when: def result = gradleRunner( - ":myserver:validateTransportVersionDefinitions", - ":myserver:generateTransportVersionDefinition", - "--name=new_name" , - "--branches=9.2" + ":myserver:validateTransportVersionDefinitions", + ":myserver:generateTransportVersionDefinition", + "--name=new_name", + "--branches=9.2" ).build() then: From 2b8c8f6bb68f206b89b92630b25f2b264f1ea4d2 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Thu, 21 Aug 2025 11:41:50 -0700 Subject: [PATCH 25/68] iter --- .../TransportVersionGenerationFuncTest.groovy | 198 ++++++++++++++++-- 1 file changed, 178 insertions(+), 20 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index 6571a31d15d2b..268629874f01a 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -95,7 +95,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes when: "We modify the latest files" originalModifiedLatestFiles.forEach { - latestTransportVersion(it.branch, it.name + "_modification", (it.id + 1).toString()) + latestTransportVersion(it.branch, it.name + "_modification", (it.id + 7).toString()) } and: "We run the generation task" @@ -106,10 +106,12 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes if (branchesParam != null) { args.add("--branches=" + branchesParam.join(",")) } + def result = gradleRunner(args.toArray(new String[0])).build() + + and: "If a name is specified, we add a reference so validation succeeds" if (name != null) { - referencedTransportVersion(name) // Ensure the definition file is referenced to allow validation to succeed + referencedTransportVersion(name) } - def result = gradleRunner(args.toArray(new String[0])).build() then: "The generation and validation tasks should succeed" result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS @@ -143,7 +145,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes } // TODO this test is finding a legitimate bug - def "definitions that are no longer referenced should be deleted and latest files reverted"(List branches) { + def "When a reference is removed after a definition is generated, the definition should be deleted and latest files reverted"(List branches) { given: String definitionName = "test_tv_patch_ids" referencedTransportVersion(definitionName) @@ -182,34 +184,190 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes ] } - def "unreferenced definition files should be deleted"() { + def "When a reference is renamed after a definition was generated, the original should be removed and latest files updated"(List branches) { given: - namedTransportVersion("old_name", "8124000") - referencedTransportVersion("new_name") + String firstName = "original_tv_name" + referencedTransportVersion(firstName) + List originalLatestFiles1 = readLatestFiles(branches) - when: + when: "The definition is generated" def result = gradleRunner( ":myserver:validateTransportVersionDefinitions", ":myserver:generateTransportVersionDefinition", - "--name=new_name", - "--branches=9.2" + "--name=" + firstName, + "--branches=" + branches.join(",") ).build() - then: - !file("myserver/src/main/resources/transport/definitions/named/old_name.csv").exists() + then: "The generation task should succeed and create the definition file" + file("myserver/src/main/resources/transport/definitions/named/${firstName}.csv").exists() result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + validateDefinitionFile(firstName, originalLatestFiles1) + + when: "The reference is renamed and the generation is run again" + deleteTransportVersionReference(firstName) + String secondName = "new_tv_name" + referencedTransportVersion(secondName) + List originalLatestFiles2 = readLatestFiles(branches) + + def secondResult = gradleRunner( + ":myserver:validateTransportVersionDefinitions", + ":myserver:generateTransportVersionDefinition", + "--branches=" + branches.join(",") + ).build() + + then: "The generation task should succeed and the definition file should be deleted" + !file("myserver/src/main/resources/transport/definitions/named/${firstName}.csv").exists() + file("myserver/src/main/resources/transport/definitions/named/${secondName}.csv").exists() + secondResult.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + secondResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + validateDefinitionFile(secondName, originalLatestFiles2) + + where: + branches << [ + ["9.2"], + ["9.2", "9.1"] + ] } + def "when a definition file is deleted and the reference and latest files haven't been, the system should regenerate"(List branches) { + given: + String definitionName = "test_tv" + referencedTransportVersion(definitionName) + List originalLatestFiles = readLatestFiles(branches) + + when: "The definition is generated" + def result = gradleRunner( + ":myserver:validateTransportVersionDefinitions", + ":myserver:generateTransportVersionDefinition", + "--name=" + definitionName, + "--branches=" + branches.join(",") + ).build() + + then: "The generation task should succeed and create the definition file" + result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + validateDefinitionFile(definitionName, originalLatestFiles) + + when: "The definition file is deleted" + file("myserver/src/main/resources/transport/definitions/named/${definitionName}.csv").delete() + + then: "The definition file should no longer exist" + !file("myserver/src/main/resources/transport/definitions/named/${definitionName}.csv").exists() + + when: "Validation is run" + def validationResult = gradleRunner(":myserver:validateTransportVersionDefinitions").buildAndFail() + + then: "The validation task should fail since the definition file is missing" + validationResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.FAILED + + when: "The generation task is run again" + def secondResult = gradleRunner( + ":myserver:validateTransportVersionDefinitions", + ":myserver:generateTransportVersionDefinition", + "--branches=" + branches.join(",") + ).build() + + then: "The generation task should succeed and the definition file should be recreated" + secondResult.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + secondResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + validateDefinitionFile(definitionName, originalLatestFiles) + + where: + branches << [ + ["9.2"], + ["9.2", "9.1"] + ] + } - /* - TODO: Add tests that check that: - - - - a latest file without a corresponding definition file should be reverted to main - - a merge conflict should be resolved, resulting in regeneration of the latest file. - - add a reference, generate, remove the reference, add another reference, generate - - - */ + def "When a latest file is incorrectly changed and a referenced definition file exists, the latest file should be regenerated"(List branches) { + given: + String definitionName = "test_tv" + referencedTransportVersion(definitionName) + List originalLatestFiles = readLatestFiles(branches) + + when: "The definition is generated" + def result = gradleRunner( + ":myserver:validateTransportVersionDefinitions", + ":myserver:generateTransportVersionDefinition", + "--name=" + definitionName, + "--branches=" + branches.join(",") + ).build() + + then: "The generation task should succeed and create the definition file" + result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + validateDefinitionFile(definitionName, originalLatestFiles) + + when: "The latest files are modified" + originalLatestFiles.forEach { + latestTransportVersion(it.branch, it.name + "_modification", (it.id + 7).toString()) + } + + and: "The generation task is run again" + def secondResult = gradleRunner( + ":myserver:validateTransportVersionDefinitions", + ":myserver:generateTransportVersionDefinition", + "--branches=" + branches.join(",") + ).build() + + then: "The generation task should succeed and the latest files should be reverted and incremented correctly" + secondResult.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + secondResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + validateDefinitionFile(definitionName, originalLatestFiles) + + where: + branches << [ + ["9.2"], + ["9.2", "9.1"] + ] + } + + // TODO legitimate bug, need to always clean up latest for patch versions + def "When a definition is created with a patch version, then generation is called without the patch version, the state should be updated"() { + given: + String definitionName = "test_tv" + referencedTransportVersion(definitionName) + List branches = ["9.2", "9.1"] + List mainBranch = ["9.2"] + List originalLatestFilesWithPatch = readLatestFiles(branches) + List originalLatestMainFile = readLatestFiles(mainBranch) + String originalLatestPatchText = file("myserver/src/main/resources/transport/latest/9.1.csv").text.strip() + + when: "The definition is generated with a patch version" + def result = gradleRunner( + ":myserver:validateTransportVersionDefinitions", + ":myserver:generateTransportVersionDefinition", + "--name=" + definitionName, + "--branches=" + branches.join(",") + ).build() + + then: "The generation task should succeed and create the definition file" + result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + validateDefinitionFile(definitionName, originalLatestFilesWithPatch) + + when: "The generation is called again without the patch version" + def secondResult = gradleRunner( + ":myserver:validateTransportVersionDefinitions", + ":myserver:generateTransportVersionDefinition", + "--branches=" + mainBranch.join(",") + ).build() + + then: "The generation task should succeed and the definition file should be updated" + secondResult.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + secondResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + validateDefinitionFile(definitionName, originalLatestMainFile) + originalLatestPatchText == file("myserver/src/main/resources/transport/latest/9.1.csv").text.strip() + } + + def "Latest files mangled by a merge conflict should be regenerated, and the most recent definition file should be updated"() { + + } + + def "When a reference is deleted, the system should revert to the original state"() { + + } List readLatestFiles(List branches) { return branches.stream() From 9725948d46d9c30285fb925e8a770463c83b5451 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Thu, 21 Aug 2025 12:33:07 -0700 Subject: [PATCH 26/68] more tests --- .../TransportVersionGenerationFuncTest.groovy | 79 +++++++++++++++++-- 1 file changed, 73 insertions(+), 6 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index 268629874f01a..61c86cdcf031d 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -230,7 +230,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes ] } - def "when a definition file is deleted and the reference and latest files haven't been, the system should regenerate"(List branches) { + def "when a definition file is deleted and the reference and latest files haven't been changed, the system should regenerate"(List branches) { given: String definitionName = "test_tv" referencedTransportVersion(definitionName) @@ -248,6 +248,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS validateDefinitionFile(definitionName, originalLatestFiles) + def latestFilesAfterGeneration = readLatestFiles(branches) when: "The definition file is deleted" file("myserver/src/main/resources/transport/definitions/named/${definitionName}.csv").delete() @@ -273,6 +274,14 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes secondResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS validateDefinitionFile(definitionName, originalLatestFiles) + and: "The latest files should remain unchanged" + latestFilesAfterGeneration.forEach { originalLatest -> + def latest = readLatestFile(originalLatest.branch) + assert latest.branch == originalLatest.branch + assert latest.id == originalLatest.id + assert latest.name == originalLatest.name + } + where: branches << [ ["9.2"], @@ -300,6 +309,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes validateDefinitionFile(definitionName, originalLatestFiles) when: "The latest files are modified" + List latestFilesAfterGeneration = readLatestFiles(branches) originalLatestFiles.forEach { latestTransportVersion(it.branch, it.name + "_modification", (it.id + 7).toString()) } @@ -316,6 +326,13 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes secondResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS validateDefinitionFile(definitionName, originalLatestFiles) + latestFilesAfterGeneration.forEach { originalLatest -> + def latest = readLatestFile(originalLatest.branch) + assert latest.branch == originalLatest.branch + assert latest.id == originalLatest.id + assert latest.name == originalLatest.name + } + where: branches << [ ["9.2"], @@ -324,7 +341,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes } // TODO legitimate bug, need to always clean up latest for patch versions - def "When a definition is created with a patch version, then generation is called without the patch version, the state should be updated"() { + def "When a definition is created with a patch version, then generation is called without the patch version, the latest patch file should be reverted"() { given: String definitionName = "test_tv" referencedTransportVersion(definitionName) @@ -332,7 +349,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes List mainBranch = ["9.2"] List originalLatestFilesWithPatch = readLatestFiles(branches) List originalLatestMainFile = readLatestFiles(mainBranch) - String originalLatestPatchText = file("myserver/src/main/resources/transport/latest/9.1.csv").text.strip() + LatestFile originalLatestPatchFile = readLatestFile("9.1") when: "The definition is generated with a patch version" def result = gradleRunner( @@ -358,14 +375,64 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes secondResult.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS secondResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS validateDefinitionFile(definitionName, originalLatestMainFile) - originalLatestPatchText == file("myserver/src/main/resources/transport/latest/9.1.csv").text.strip() + + and: "The latest file for the patch version should be reverted to the original state" + LatestFile latestPatchFile = readLatestFile("9.1") + latestPatchFile.id == originalLatestPatchFile.id + latestPatchFile.name == originalLatestPatchFile.name + latestPatchFile.branch == originalLatestPatchFile.branch } - def "Latest files mangled by a merge conflict should be regenerated, and the most recent definition file should be updated"() { + def "When a reference is deleted, the system should delete the definition and revert the latest files"(List branches) { + given: + String definitionName = "test_tv" + referencedTransportVersion(definitionName) + List originalLatestFiles = readLatestFiles(branches) + + when: "The definition is generated" + def result = gradleRunner( + ":myserver:validateTransportVersionDefinitions", + ":myserver:generateTransportVersionDefinition", + "--name=" + definitionName, + "--branches=" + branches.join(",") + ).build() + + then: "The generation task should succeed and create the definition file" + result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + validateDefinitionFile(definitionName, originalLatestFiles) + + when: "The reference is deleted and the generation is run again" + deleteTransportVersionReference(definitionName) + def secondResult = gradleRunner( + ":myserver:validateTransportVersionDefinitions", + ":myserver:generateTransportVersionDefinition", + "--branches=" + branches.join(",") + ).build() + + then: "The generation task should succeed and the definition file should be deleted" + !file("myserver/src/main/resources/transport/definitions/named/${definitionName}.csv").exists() + secondResult.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + secondResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + + and: "The latest files should be reverted to their original state" + originalLatestFiles.forEach { originalLatest -> + def latest = readLatestFile(originalLatest.branch) + assert latest.branch == originalLatest.branch + assert latest.id == originalLatest.id + assert latest.name == originalLatest.name + } + + where: + branches << [ + ["9.2"], + ["9.2", "9.1"] + ] } - def "When a reference is deleted, the system should revert to the original state"() { + + def "Latest files mangled by a merge conflict should be regenerated, and the most recent definition file should be updated"() { } From aa60bdb25eee6f8b39f4bf87b308beef9595f5af Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Thu, 21 Aug 2025 12:39:58 -0700 Subject: [PATCH 27/68] more tests --- .../transport/TransportVersionGenerationFuncTest.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index 61c86cdcf031d..e100677ef446d 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -433,7 +433,9 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes def "Latest files mangled by a merge conflict should be regenerated, and the most recent definition file should be updated"() { - + // TODO when will this be run? Inside the merge conflict resolution? If so, there will be two new definition files and references. + // How do we know which one to apply first? We need to use git to find the branch (main) that is being merged into. + // Will this be different than `main` when not in conflict resolution? e.g. do we need a different git command for this? } List readLatestFiles(List branches) { From 5c6a25affcf3ce0a8d56f3692ad910bf98984899 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Mon, 25 Aug 2025 11:42:09 -0700 Subject: [PATCH 28/68] iter --- .../TransportVersionGenerationFuncTest.groovy | 97 ++++++++++++++----- ...enerateTransportVersionDefinitionTask.java | 8 +- 2 files changed, 77 insertions(+), 28 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index e100677ef446d..239f2e80e066e 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -106,13 +106,13 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes if (branchesParam != null) { args.add("--branches=" + branchesParam.join(",")) } - def result = gradleRunner(args.toArray(new String[0])).build() - - and: "If a name is specified, we add a reference so validation succeeds" if (name != null) { referencedTransportVersion(name) + args.add("--name=" + name) } + def result = gradleRunner(args.toArray(new String[0])).build() + then: "The generation and validation tasks should succeed" result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS @@ -140,32 +140,23 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes ["9.2", "9.1"] | ["9.2"] | null ["9.2"] | ["9.1"] | null ["9.2"] | ["9.1"] | "test_tv" // TODO legitimate bug, need to always clean up latest. + ["9.2"] | ["9.1"] | null // TODO legitimate bug, need to always clean up latest. ["9.2", "9.1"] | ["9.2", "9.1"] | "test_tv" } + // TODO write a test that ensures all latest files are reset when the name is null, rather than have them as scenarios above + + // TODO this test is finding a legitimate bug + // TODO remove the first generation call, this can be done with helpers. def "When a reference is removed after a definition is generated, the definition should be deleted and latest files reverted"(List branches) { given: String definitionName = "test_tv_patch_ids" - referencedTransportVersion(definitionName) - List originalLatestFiles = readLatestFiles(branches) + namedTransportVersion(definitionName, "8124000") + latestTransportVersion("9.2", definitionName, "8124000") - when: "The definition is generated" - def result = gradleRunner( - ":myserver:validateTransportVersionDefinitions", - ":myserver:generateTransportVersionDefinition", - "--name=" + definitionName, - "--branches=" + branches.join(",") - ).build() - - then: "The generation task should succeed and create the definition file" - result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS - result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS - validateDefinitionFile(definitionName, originalLatestFiles) - - when: "The reference is removed and the generation is run again" - deleteTransportVersionReference(definitionName) + when: "generation is run" def secondResult = gradleRunner( ":myserver:validateTransportVersionDefinitions", ":myserver:generateTransportVersionDefinition", @@ -180,7 +171,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes where: branches << [ ["9.2"], - ["9.2", "9.1"] +// ["9.2", "9.1"] // TODO add another test for this, since this is a different setup ] } @@ -431,13 +422,69 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes ] } + // TODO when will this be run? Inside the merge conflict resolution? If so, there will be two new definition files and references. + // How do we know which one to apply first? We need to use git to find the branch (main) that is being merged into. + // Will this be different than `main` when not in conflict resolution? e.g. do we need a different git command for this? + def "Latest files mangled by a merge conflict should be regenerated, and the most recent definition file should be updated"(List branches) { + given: + String firstDefinitionName = "first_tv" + String secondDefinitionName = "second_tv" + List originalLatestFiles = readLatestFiles(branches) + + when: "Two definitions are created and generated from the same latest files" + def originalMainLatest = originalLatestFiles.first() + definedAndUsedTransportVersion(firstDefinitionName, (originalMainLatest.getId() + 1000).toString()) + definedAndUsedTransportVersion(secondDefinitionName, (originalMainLatest.getId() + 1000).toString()) + + if (branches.size() > 1) { + LatestFile originalPatchLatest = originalLatestFiles.last() + definedAndUsedTransportVersion(firstDefinitionName, (originalPatchLatest.getId() + 1).toString()) + definedAndUsedTransportVersion(secondDefinitionName, (originalPatchLatest.getId() + 1).toString()) + } + + and: "Mangled latest files are produced from a git conflict" + file("myserver/src/main/resources/transport/latest/${originalMainLatest.branch}.csv").text = + """ + <<<<<<< HEAD\n" + + "${firstDefinitionName},${originalMainLatest.id + 1000} + ======= + ${secondDefinitionName},${originalMainLatest.id + 1000} + >>>>>> branch + """.strip() + + if (branches.size() > 1) { + LatestFile originalPatchLatest = originalLatestFiles.last() + file("myserver/src/main/resources/transport/latest/${originalPatchLatest.branch}.csv").text = + """ + <<<<<<< HEAD\n" + + "${firstDefinitionName},${originalPatchLatest.id + 1} + ======= + ${secondDefinitionName},${originalPatchLatest.id + 1} + >>>>>> branch + """.strip() + } - def "Latest files mangled by a merge conflict should be regenerated, and the most recent definition file should be updated"() { - // TODO when will this be run? Inside the merge conflict resolution? If so, there will be two new definition files and references. - // How do we know which one to apply first? We need to use git to find the branch (main) that is being merged into. - // Will this be different than `main` when not in conflict resolution? e.g. do we need a different git command for this? + and: "The generation task is run" + def result = gradleRunner( + ":myserver:validateTransportVersionDefinitions", + ":myserver:generateTransportVersionDefinition", + "--branches=" + branches.join(",") + ).build() + + then: "The generation task should succeed and the latest files should be reverted and incremented correctly" + result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + validateDefinitionFile(secondDefinitionName, originalLatestFiles) + + where: + branches << [ + ["9.2"], + ["9.2", "9.1"] + ] } + // TODO do we need a test that has a garbled latest file but Main is OK, and it increments correctly? + List readLatestFiles(List branches) { return branches.stream() .map { readLatestFile(it) } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index 4b959946df8ba..fb6fb06d9cd18 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -24,6 +24,7 @@ import org.gradle.process.ExecOperations; import org.gradle.process.ExecResult; +import javax.inject.Inject; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.file.Path; @@ -35,8 +36,6 @@ import java.util.Set; import java.util.stream.Collectors; -import javax.inject.Inject; - /** * This task generates transport version definition files. These files * are runtime resources that TransportVersion loads statically. @@ -67,7 +66,7 @@ public Path getResourcesDir() { */ @Input @Optional - @Option(option = "name", description = "TBD") + @Option(option = "name", description = "TBD") // TODO add description public abstract Property getTransportVersionName(); // The plugin should always set this, not optional /** @@ -106,6 +105,7 @@ public void run() throws IOException { : findAddedTransportVersionName(resources, referencedNames, changedDefinitionNames); if (name.isEmpty()) { + // Todo this should reset all the changed latest files regardless of if there's a name or not resetAllLatestFiles(resources); } else { List ids = updateLatestFiles(resources, name); @@ -137,6 +137,8 @@ private List updateLatestFiles(TransportVersionResourcesServ TransportVersionId id = TransportVersionId.fromInt(mainLatest.id().complete() + increment); ids.add(id); resources.writeLatestFile(new TransportVersionLatest(mainLatest.releaseBranch(), name, id)); + } else { + resources.writeLatestFile(mainLatest); } } } From 3af734227f3c02e13f9b397822c8fdf35692b4ab Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 25 Aug 2025 18:49:20 +0000 Subject: [PATCH 29/68] [CI] Auto commit changes from spotless --- .../transport/GenerateTransportVersionDefinitionTask.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index fb6fb06d9cd18..82cc60b53fb6d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -24,7 +24,6 @@ import org.gradle.process.ExecOperations; import org.gradle.process.ExecResult; -import javax.inject.Inject; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.file.Path; @@ -36,6 +35,8 @@ import java.util.Set; import java.util.stream.Collectors; +import javax.inject.Inject; + /** * This task generates transport version definition files. These files * are runtime resources that TransportVersion loads statically. From b978bbc1aa32b658a8a965f53cebbb8282470cff Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 25 Aug 2025 12:30:27 -0700 Subject: [PATCH 30/68] test iters --- .../TransportVersionGenerationFuncTest.groovy | 86 +++++-------------- ...enerateTransportVersionDefinitionTask.java | 7 +- .../TransportVersionResourcesService.java | 2 + 3 files changed, 28 insertions(+), 67 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index 239f2e80e066e..6cf5635d6705e 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -178,29 +178,12 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes def "When a reference is renamed after a definition was generated, the original should be removed and latest files updated"(List branches) { given: String firstName = "original_tv_name" - referencedTransportVersion(firstName) - List originalLatestFiles1 = readLatestFiles(branches) - - when: "The definition is generated" - def result = gradleRunner( - ":myserver:validateTransportVersionDefinitions", - ":myserver:generateTransportVersionDefinition", - "--name=" + firstName, - "--branches=" + branches.join(",") - ).build() - - then: "The generation task should succeed and create the definition file" - file("myserver/src/main/resources/transport/definitions/named/${firstName}.csv").exists() - result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS - result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS - validateDefinitionFile(firstName, originalLatestFiles1) - - when: "The reference is renamed and the generation is run again" - deleteTransportVersionReference(firstName) + namedTransportVersion(firstName, "8124000") + latestTransportVersion("9.2", firstName, "8124000") String secondName = "new_tv_name" referencedTransportVersion(secondName) - List originalLatestFiles2 = readLatestFiles(branches) + when: def secondResult = gradleRunner( ":myserver:validateTransportVersionDefinitions", ":myserver:generateTransportVersionDefinition", @@ -212,7 +195,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes file("myserver/src/main/resources/transport/definitions/named/${secondName}.csv").exists() secondResult.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS secondResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS - validateDefinitionFile(secondName, originalLatestFiles2) + file("myserver/src/main/resources/transport/latest/9.2.csv").text.strip() == "new_tv_name,8124000" where: branches << [ @@ -425,62 +408,33 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes // TODO when will this be run? Inside the merge conflict resolution? If so, there will be two new definition files and references. // How do we know which one to apply first? We need to use git to find the branch (main) that is being merged into. // Will this be different than `main` when not in conflict resolution? e.g. do we need a different git command for this? - def "Latest files mangled by a merge conflict should be regenerated, and the most recent definition file should be updated"(List branches) { + def "Latest files mangled by a merge conflict should be regenerated, and the most recent definition file should be updated"() { given: - String firstDefinitionName = "first_tv" - String secondDefinitionName = "second_tv" - List originalLatestFiles = readLatestFiles(branches) - - when: "Two definitions are created and generated from the same latest files" - def originalMainLatest = originalLatestFiles.first() - definedAndUsedTransportVersion(firstDefinitionName, (originalMainLatest.getId() + 1000).toString()) - definedAndUsedTransportVersion(secondDefinitionName, (originalMainLatest.getId() + 1000).toString()) + file("myserver/src/main/resources/transport/latest/9.2.csv").text = + """ + <<<<<<< HEAD + existing_92,8123000 + ======= + second_tv,8123000 + >>>>>> branch + """.strip() - if (branches.size() > 1) { - LatestFile originalPatchLatest = originalLatestFiles.last() - definedAndUsedTransportVersion(firstDefinitionName, (originalPatchLatest.getId() + 1).toString()) - definedAndUsedTransportVersion(secondDefinitionName, (originalPatchLatest.getId() + 1).toString()) - } + definedAndUsedTransportVersion("second_tv", "8123000") - and: "Mangled latest files are produced from a git conflict" - file("myserver/src/main/resources/transport/latest/${originalMainLatest.branch}.csv").text = - """ - <<<<<<< HEAD\n" + - "${firstDefinitionName},${originalMainLatest.id + 1000} - ======= - ${secondDefinitionName},${originalMainLatest.id + 1000} - >>>>>> branch - """.strip() - - if (branches.size() > 1) { - LatestFile originalPatchLatest = originalLatestFiles.last() - file("myserver/src/main/resources/transport/latest/${originalPatchLatest.branch}.csv").text = - """ - <<<<<<< HEAD\n" + - "${firstDefinitionName},${originalPatchLatest.id + 1} - ======= - ${secondDefinitionName},${originalPatchLatest.id + 1} - >>>>>> branch - """.strip() - } - - and: "The generation task is run" + when: def result = gradleRunner( ":myserver:validateTransportVersionDefinitions", ":myserver:generateTransportVersionDefinition", - "--branches=" + branches.join(",") + "--name=second_tv", + "--branches=9.2" ).build() then: "The generation task should succeed and the latest files should be reverted and incremented correctly" result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS - validateDefinitionFile(secondDefinitionName, originalLatestFiles) - - where: - branches << [ - ["9.2"], - ["9.2", "9.1"] - ] + file("myserver/src/main/resources/transport/definitions/named/existing_92.csv").text.strip() == "8123000,8012001" + file("myserver/src/main/resources/transport/definitions/named/second_tv.csv").text.strip() == "8124000" + file("myserver/src/main/resources/transport/latest/9.2.csv").text.strip() == "second_tv,8124000" } // TODO do we need a test that has a garbled latest file but Main is OK, and it increments correctly? diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index 82cc60b53fb6d..182e1c3379a5a 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -228,7 +228,12 @@ private String findAddedTransportVersionName( if (changedDefinitions.isEmpty()) { return ""; } else { - return changedDefinitions.getFirst(); + String changedDefinitionName = changedDefinitions.getFirst(); + if (referencedNames.contains(changedDefinitionName)) { + return changedDefinitionName; + } else { + return ""; // the changed name is unreferenced, so go into "reset mode" + } } } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java index 689a46027a3ec..b65debe84cc22 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java @@ -147,6 +147,7 @@ Path getRepositoryPath(TransportVersionDefinition definition) { } void writeNamedDefinition(TransportVersionDefinition definition) throws IOException { + System.out.println("Writing definition: " + definition); Path path = transportResourcesDir.resolve(getNamedDefinitionRelativePath(definition.name())); Files.writeString( path, @@ -198,6 +199,7 @@ Path getRepositoryPath(TransportVersionLatest latest) { } void writeLatestFile(TransportVersionLatest latest) throws IOException { + System.out.println("Writing latest file: " + latest); Path path = transportResourcesDir.resolve(getLatestRelativePath(latest.releaseBranch())); Files.writeString(path, latest.name() + "," + latest.id().complete() + "\n", StandardCharsets.UTF_8); } From 961f0b862c297ea227fdae0975fe63f180b20f5d Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Mon, 25 Aug 2025 13:38:01 -0700 Subject: [PATCH 31/68] iter --- .../TransportVersionGenerationFuncTest.groovy | 44 ++++++++++++++++--- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index 6cf5635d6705e..b25d667d78da1 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -135,18 +135,52 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes where: branchesParam | latestFilesModified | name - null | ["9.2"] | null - null | ["9.2", "9.1"] | null - ["9.2", "9.1"] | ["9.2"] | null - ["9.2"] | ["9.1"] | null ["9.2"] | ["9.1"] | "test_tv" // TODO legitimate bug, need to always clean up latest. ["9.2"] | ["9.1"] | null // TODO legitimate bug, need to always clean up latest. ["9.2", "9.1"] | ["9.2", "9.1"] | "test_tv" } - // TODO write a test that ensures all latest files are reset when the name is null, rather than have them as scenarios above + // TODO write a test that ensures all latest files are reset when the name is null, rather than having them as scenarios above + def "Irrelevant changes to the latest file should be reverted"() { + when: "We modify the latest files" + file("myserver/src/main/resources/transport/latest/9.2.csv").text = "modification,9000000" + + and: "We run the generation task with no name specified" + def result = gradleRunner( + ":myserver:validateTransportVersionDefinitions", + ":myserver:generateTransportVersionDefinition", + "--branches=9.2" + ).build() + + then: "The generation and validation tasks should succeed" + result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + + and: "The latest file should be reverted to its original state on main" + file("myserver/src/main/resources/transport/latest/9.2.csv").text.strip() == "existing_92,8123000" + } + def "Irrelevant changes to multiple latest files should be reverted"() { + when: "We modify the latest files" + file("myserver/src/main/resources/transport/latest/9.2.csv").text = "modification,9000000" + file("myserver/src/main/resources/transport/latest/9.1.csv").text = "modification,9000000" + + and: "We run the generation task with no name specified" + def result = gradleRunner( + ":myserver:validateTransportVersionDefinitions", + ":myserver:generateTransportVersionDefinition", + "--branches=9.2,9.1" + ).build() + + then: "The generation and validation tasks should succeed" + result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + + and: "The latest files should be reverted to their original state" + file("myserver/src/main/resources/transport/latest/9.2.csv").text.strip() == "existing_92,8123000" + file("myserver/src/main/resources/transport/latest/9.1.csv").text.strip() == "existing_92,8012001" + } // TODO this test is finding a legitimate bug // TODO remove the first generation call, this can be done with helpers. From f68236da4e500729d01a8053088748ac0792fa31 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 25 Aug 2025 14:06:28 -0700 Subject: [PATCH 32/68] iter --- .../AbstractTransportVersionFuncTest.groovy | 12 ++-- .../TransportVersionGenerationFuncTest.groovy | 66 +++++++++---------- 2 files changed, 41 insertions(+), 37 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy index 4b85ce0a8ee99..bfb755f7ca484 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy @@ -84,16 +84,20 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest { return gradleRunner(":myserver:validateTransportVersionDefinitions").buildAndFail() } - def assertReferencesFailure(BuildResult result, String project, String expectedOutput) { - result.task(":${project}:validateTransportVersionReferences").outcome == TaskOutcome.FAILED + void assertReferencesFailure(BuildResult result, String project, String expectedOutput) { + assert result.task(":${project}:validateTransportVersionReferences").outcome == TaskOutcome.FAILED assertOutputContains(result.output, expectedOutput) } - def assertDefinitionsFailure(BuildResult result, String expectedOutput) { - result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.FAILED + void assertDefinitionsFailure(BuildResult result, String expectedOutput) { + assert result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.FAILED assertOutputContains(result.output, expectedOutput) } + void assertLatest(String releaseBranch, String content) { + assert file("myserver/src/main/resources/transport/latest/${releaseBranch}.csv").text.strip() == content + } + def setup() { configurationCacheCompatible = false internalBuild() diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index b25d667d78da1..fd8b094f057f1 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -9,9 +9,24 @@ package org.elasticsearch.gradle.internal.transport +import org.gradle.testkit.runner.BuildResult import org.gradle.testkit.runner.TaskOutcome class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTest { + + def runGenerateAndValidateTask(String... additionalArgs) { + List args = new ArrayList<>() + args.add(":myserver:validateTransportVersionDefinitions") + args.add(":myserver:generateTransportVersionDefinition") + args.addAll(additionalArgs); + return gradleRunner(args.toArray()).build() + } + + void assertGenerateAndValidateSuccess(BuildResult result) { + assert result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + assert result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + } + def "test setup works"() { when: def result = gradleRunner( @@ -141,45 +156,30 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes } - // TODO write a test that ensures all latest files are reset when the name is null, rather than having them as scenarios above - def "Irrelevant changes to the latest file should be reverted"() { - when: "We modify the latest files" - file("myserver/src/main/resources/transport/latest/9.2.csv").text = "modification,9000000" - - and: "We run the generation task with no name specified" - def result = gradleRunner( - ":myserver:validateTransportVersionDefinitions", - ":myserver:generateTransportVersionDefinition", - "--branches=9.2" - ).build() + def "irrelevant changes to the latest file should be reverted"() { + given: + latestTransportVersion("9.2", "modification", "9000000") - then: "The generation and validation tasks should succeed" - result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS - result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + when: + def result = runGenerateAndValidateTask("--branches=9.2") - and: "The latest file should be reverted to its original state on main" - file("myserver/src/main/resources/transport/latest/9.2.csv").text.strip() == "existing_92,8123000" + then: + assertGenerateAndValidateSuccess(result) + assertLatest("9.2", "existing_92,8123000") } - def "Irrelevant changes to multiple latest files should be reverted"() { - when: "We modify the latest files" - file("myserver/src/main/resources/transport/latest/9.2.csv").text = "modification,9000000" - file("myserver/src/main/resources/transport/latest/9.1.csv").text = "modification,9000000" - - and: "We run the generation task with no name specified" - def result = gradleRunner( - ":myserver:validateTransportVersionDefinitions", - ":myserver:generateTransportVersionDefinition", - "--branches=9.2,9.1" - ).build() + def "irrelevant changes to multiple latest files should be reverted"() { + given: + latestTransportVersion("9.2", "modification", "9000000") + latestTransportVersion("9.1", "modification", "9000000") - then: "The generation and validation tasks should succeed" - result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS - result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + when: + def result = runGenerateAndValidateTask("--branches=9.2,9.1") - and: "The latest files should be reverted to their original state" - file("myserver/src/main/resources/transport/latest/9.2.csv").text.strip() == "existing_92,8123000" - file("myserver/src/main/resources/transport/latest/9.1.csv").text.strip() == "existing_92,8012001" + then: + assertGenerateAndValidateSuccess(result) + assertLatest("9.2", "existing_92,8123000") + assertLatest("9.1", "existing_92,8012001") } // TODO this test is finding a legitimate bug From ab7e60588bdd9e823fd55596353f0940b9b4304e Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Mon, 25 Aug 2025 14:12:21 -0700 Subject: [PATCH 33/68] iter --- .../TransportVersionGenerationFuncTest.groovy | 26 +++++++------------ 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index fd8b094f057f1..a9886dfe5cc3a 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -39,7 +39,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS } - def "A definition should be generated when specified by an arg but no code reference exists"(List branches) { + def "a definition should be generated when specified by an arg but no code reference exists"(List branches) { given: String tvName = "test_tv_patch_ids" List latestBranchesToOriginalIds = readLatestFiles(branches) @@ -69,7 +69,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes ] } - def "A definition should be generated when the name arg isn't specified but a code reference exists"() { + def "a definition should be generated when the name arg isn't specified but a code reference exists"() { given: String tvName = "test_tv_patch_ids" referencedTransportVersion(tvName) @@ -89,7 +89,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes validateDefinitionFile(tvName, latestBranchesToOriginalIds) } - def "Generation should fail if the branches arg is omitted, the state should remain unaltered"() { + def "generation should fail if the branches arg is omitted, the state should remain unaltered"() { when: def generateResult = gradleRunner("generateTransportVersionDefinition", "--name=no_branches").buildAndFail() def validateResult = gradleRunner("validateTransportVersionDefinitions").build() @@ -99,7 +99,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes validateResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS } - def "Latest file modifications should be reverted to their original state on main"( + def "latest file modifications should be reverted to their original state on main"( List branchesParam, List latestFilesModified, String name @@ -182,9 +182,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertLatest("9.1", "existing_92,8012001") } - // TODO this test is finding a legitimate bug - // TODO remove the first generation call, this can be done with helpers. - def "When a reference is removed after a definition is generated, the definition should be deleted and latest files reverted"(List branches) { + def "when a reference is removed after a definition is generated, the definition should be deleted and latest files reverted"(List branches) { given: String definitionName = "test_tv_patch_ids" namedTransportVersion(definitionName, "8124000") @@ -209,7 +207,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes ] } - def "When a reference is renamed after a definition was generated, the original should be removed and latest files updated"(List branches) { + def "when a reference is renamed after a definition was generated, the original should be removed and latest files updated"(List branches) { given: String firstName = "original_tv_name" namedTransportVersion(firstName, "8124000") @@ -297,7 +295,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes ] } - def "When a latest file is incorrectly changed and a referenced definition file exists, the latest file should be regenerated"(List branches) { + def "when a latest file is incorrectly changed and a referenced definition file exists, the latest file should be regenerated"(List branches) { given: String definitionName = "test_tv" referencedTransportVersion(definitionName) @@ -348,8 +346,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes ] } - // TODO legitimate bug, need to always clean up latest for patch versions - def "When a definition is created with a patch version, then generation is called without the patch version, the latest patch file should be reverted"() { + def "when a definition is created with a patch version, then generation is called without the patch version, the latest patch file should be reverted"() { given: String definitionName = "test_tv" referencedTransportVersion(definitionName) @@ -392,7 +389,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes } - def "When a reference is deleted, the system should delete the definition and revert the latest files"(List branches) { + def "when a reference is deleted, the system should delete the definition and revert the latest files"(List branches) { given: String definitionName = "test_tv" referencedTransportVersion(definitionName) @@ -439,10 +436,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes ] } - // TODO when will this be run? Inside the merge conflict resolution? If so, there will be two new definition files and references. - // How do we know which one to apply first? We need to use git to find the branch (main) that is being merged into. - // Will this be different than `main` when not in conflict resolution? e.g. do we need a different git command for this? - def "Latest files mangled by a merge conflict should be regenerated, and the most recent definition file should be updated"() { + def "latest files mangled by a merge conflict should be regenerated, and the most recent definition file should be updated"() { given: file("myserver/src/main/resources/transport/latest/9.2.csv").text = """ From 7e74258c4d62f58be132b2591cf1f05553214197 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Mon, 25 Aug 2025 14:26:38 -0700 Subject: [PATCH 34/68] iter --- .../TransportVersionGenerationFuncTest.groovy | 43 +++++++++++-------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index a9886dfe5cc3a..686bb687b16d1 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -182,29 +182,34 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertLatest("9.1", "existing_92,8012001") } - def "when a reference is removed after a definition is generated, the definition should be deleted and latest files reverted"(List branches) { + def "state should be reverted when a reference is removed"() { given: - String definitionName = "test_tv_patch_ids" - namedTransportVersion(definitionName, "8124000") - latestTransportVersion("9.2", definitionName, "8124000") + namedTransportVersion("test_tv", "8124000") + latestTransportVersion("9.2", "test_tv", "8124000") - when: "generation is run" - def secondResult = gradleRunner( - ":myserver:validateTransportVersionDefinitions", - ":myserver:generateTransportVersionDefinition", - "--branches=" + branches.join(",") - ).build() + when: + def result = runGenerateAndValidateTask("--branches=9.2") - then: "The generation task should succeed and the definition file should be deleted" - !file("myserver/src/main/resources/transport/definitions/named/${definitionName}.csv").exists() - secondResult.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS - secondResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + then: + assertGenerateAndValidateSuccess(result) + !file("myserver/src/main/resources/transport/definitions/named/test_tv.csv").exists() + assertLatest("9.2", "existing_92,8123000") + } - where: - branches << [ - ["9.2"], -// ["9.2", "9.1"] // TODO add another test for this, since this is a different setup - ] + def "state on multiple latest files should be reverted when a reference is removed"() { + given: + namedTransportVersion("test_tv", "8124000") + latestTransportVersion("9.2", "test_tv", "8124000") + latestTransportVersion("9.1", "test_tv", "8012002") + + when: + def result = runGenerateAndValidateTask("--branches=9.2") + + then: + assertGenerateAndValidateSuccess(result) + !file("myserver/src/main/resources/transport/definitions/named/test_tv.csv").exists() + assertLatest("9.2", "existing_92,8123000") + assertLatest("9.1", "existing_92,8012001") } def "when a reference is renamed after a definition was generated, the original should be removed and latest files updated"(List branches) { From 1b3629be6e6666b989b92e784b5397d3ce8576ff Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Mon, 25 Aug 2025 15:00:39 -0700 Subject: [PATCH 35/68] iter --- .../TransportVersionGenerationFuncTest.groovy | 51 ++++++++++--------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index 686bb687b16d1..97138625f2db6 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -212,36 +212,41 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertLatest("9.1", "existing_92,8012001") } - def "when a reference is renamed after a definition was generated, the original should be removed and latest files updated"(List branches) { + def "a reference can be renamed"() { given: - String firstName = "original_tv_name" - namedTransportVersion(firstName, "8124000") - latestTransportVersion("9.2", firstName, "8124000") - String secondName = "new_tv_name" - referencedTransportVersion(secondName) + namedTransportVersion("first_tv", "8124000") + latestTransportVersion("9.2", "first_tv", "8124000") + referencedTransportVersion("renamed_tv") when: - def secondResult = gradleRunner( - ":myserver:validateTransportVersionDefinitions", - ":myserver:generateTransportVersionDefinition", - "--branches=" + branches.join(",") - ).build() + def result = runGenerateAndValidateTask("--branches=9.2") - then: "The generation task should succeed and the definition file should be deleted" - !file("myserver/src/main/resources/transport/definitions/named/${firstName}.csv").exists() - file("myserver/src/main/resources/transport/definitions/named/${secondName}.csv").exists() - secondResult.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS - secondResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS - file("myserver/src/main/resources/transport/latest/9.2.csv").text.strip() == "new_tv_name,8124000" + then: + assertGenerateAndValidateSuccess(result) + !file("myserver/src/main/resources/transport/definitions/named/first_tv.csv").exists() + file("myserver/src/main/resources/transport/definitions/named/renamed_tv.csv").exists() + assertLatest("9.2", "renamed_tv,8124000") + } - where: - branches << [ - ["9.2"], - ["9.2", "9.1"] - ] + def "a reference with a patch version can be renamed"() { + given: + namedTransportVersion("first_tv", "8124000") + latestTransportVersion("9.2", "first_tv", "8124000") + latestTransportVersion("9.1", "first_tv", "8012002") + referencedTransportVersion("renamed_tv") + + when: + def result = runGenerateAndValidateTask("--branches=9.2,9.1") + + then: + assertGenerateAndValidateSuccess(result) + !file("myserver/src/main/resources/transport/definitions/named/first_tv.csv").exists() + file("myserver/src/main/resources/transport/definitions/named/renamed_tv.csv").exists() + assertLatest("9.2", "renamed_tv,8124000") + assertLatest("9.1", "renamed_tv,8012002") } - def "when a definition file is deleted and the reference and latest files haven't been changed, the system should regenerate"(List branches) { + def "The system should regenerate a deleted definition file"(List branches) { given: String definitionName = "test_tv" referencedTransportVersion(definitionName) From c20a191ebd0f87e42f414327e0ef1d70d4fc18d4 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 25 Aug 2025 17:35:26 -0700 Subject: [PATCH 36/68] update tests --- .../AbstractTransportVersionFuncTest.groovy | 10 + .../TransportVersionGenerationFuncTest.groovy | 540 +++++------------- ...enerateTransportVersionDefinitionTask.java | 2 +- .../TransportVersionResourcesService.java | 8 +- 4 files changed, 156 insertions(+), 404 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy index bfb755f7ca484..043b45b82e5a6 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy @@ -94,6 +94,16 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest { assertOutputContains(result.output, expectedOutput) } + void assertNamedDefinition(String name, String content) { + File definitionFile = file("myserver/src/main/resources/transport/definitions/named/${name}.csv") + assert definitionFile.exists() + assert definitionFile.text.strip() == content + } + + void assertNamedDefinitionDoesNotExist(String name) { + assert file("myserver/src/main/resources/transport/definitions/named/${name}.csv").exists() == false + } + void assertLatest(String releaseBranch, String content) { assert file("myserver/src/main/resources/transport/latest/${releaseBranch}.csv").text.strip() == content } diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index 97138625f2db6..ec1a562282745 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -19,162 +19,105 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes args.add(":myserver:validateTransportVersionDefinitions") args.add(":myserver:generateTransportVersionDefinition") args.addAll(additionalArgs); - return gradleRunner(args.toArray()).build() + return gradleRunner(args.toArray()) } - void assertGenerateAndValidateSuccess(BuildResult result) { + def runGenerateTask(String... additionalArgs) { + List args = new ArrayList<>() + args.add(":myserver:generateTransportVersionDefinition") + args.addAll(additionalArgs); + return gradleRunner(args.toArray()) + } + + def runValidateTask() { + return gradleRunner(":myserver:validateTransportVersionDefinitions") + } + + void assertGenerateSuccess(BuildResult result) { assert result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS + } + + void assertGenerateFailure(BuildResult result) { + assert result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.FAILED + } + + void assertValidateSuccess(BuildResult result) { assert result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS } - def "test setup works"() { + void assertGenerateAndValidateSuccess(BuildResult result) { + assertGenerateSuccess(result) + assertValidateSuccess(result) + } + + def "setup is valid"() { when: - def result = gradleRunner( - ":myserver:generateTransportVersionDefinition", - ":myserver:validateTransportVersionDefinitions" - ).build() + def result = runGenerateAndValidateTask().build() then: - result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS - result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + assertGenerateAndValidateSuccess(result) } - def "a definition should be generated when specified by an arg but no code reference exists"(List branches) { - given: - String tvName = "test_tv_patch_ids" - List latestBranchesToOriginalIds = readLatestFiles(branches) - - when: "generation is run with a name specified and no code references" - def result = gradleRunner( - "generateTransportVersionDefinition", - "--name=" + tvName, - "--branches=" + branches.join(",") - ).build() - - then: "The generation task should succeed and create the definition file" - result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS - validateDefinitionFile(tvName, latestBranchesToOriginalIds) - - when: "A reference is added" - referencedTransportVersion(tvName) - def validateResult = gradleRunner("validateTransportVersionDefinitions").build() - - then: "The full validation should succeed now that the reference exists" - validateResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS - - where: - branches << [ - ["9.2"], - ["9.2", "9.1"] - ] - } + def "a definition should be generated when specified by an arg but no code reference exists yet"() { + when: + def result = runGenerateTask("--name=new_tv", "--branches=9.2").build() - def "a definition should be generated when the name arg isn't specified but a code reference exists"() { - given: - String tvName = "test_tv_patch_ids" - referencedTransportVersion(tvName) - List branches = ["9.2"] - List latestBranchesToOriginalIds = readLatestFiles(branches) + then: + assertGenerateSuccess(result) + assertNamedDefinition("new_tv", "8124000") + assertLatest("9.2", "new_tv,8124000") when: - def result = gradleRunner( - ":myserver:validateTransportVersionDefinitions", - "generateTransportVersionDefinition", - "--branches=" + branches.join(",") - ).build() + referencedTransportVersion("new_tv") + def validateResult = runValidateTask().build() then: - result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS - result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS - validateDefinitionFile(tvName, latestBranchesToOriginalIds) + assertValidateSuccess(validateResult) } - def "generation should fail if the branches arg is omitted, the state should remain unaltered"() { + def "a definition should be generated when only a code reference exists"() { + given: + referencedTransportVersion("new_tv") + when: - def generateResult = gradleRunner("generateTransportVersionDefinition", "--name=no_branches").buildAndFail() - def validateResult = gradleRunner("validateTransportVersionDefinitions").build() + def result = runGenerateAndValidateTask("--branches=9.2").build() then: - generateResult.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.FAILED - validateResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS + assertGenerateAndValidateSuccess(result) + assertNamedDefinition("new_tv", "8124000") + assertLatest("9.2", "new_tv,8124000") } - def "latest file modifications should be reverted to their original state on main"( - List branchesParam, - List latestFilesModified, - String name - ) { - given: - List originalModifiedLatestFiles = readLatestFiles(latestFilesModified) - List originalBranchesLatestFiles = readLatestFiles(branchesParam) - - when: "We modify the latest files" - originalModifiedLatestFiles.forEach { - latestTransportVersion(it.branch, it.name + "_modification", (it.id + 7).toString()) - } - - and: "We run the generation task" - def args = [ - ":myserver:validateTransportVersionDefinitions", - ":myserver:generateTransportVersionDefinition" - ] - if (branchesParam != null) { - args.add("--branches=" + branchesParam.join(",")) - } - if (name != null) { - referencedTransportVersion(name) - args.add("--name=" + name) - } - - def result = gradleRunner(args.toArray(new String[0])).build() - - then: "The generation and validation tasks should succeed" - result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS - result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS - - and: "The modified latest files should be reverted if there is no name specified or they are not specified in the branches param" - originalModifiedLatestFiles.forEach { originalLatest -> - boolean noNameSpecified = name == null - boolean modifiedNotInBranchesParam = branchesParam != null && branchesParam.contains(originalLatest.branch) == false - if (noNameSpecified || modifiedNotInBranchesParam) { - def latest = readLatestFile(originalLatest.branch) - assert latest.branch == originalLatest.branch - assert latest.id == originalLatest.id - } - } - - and: "The latest files for the branches param should be incremented correctly" - if (name != null) { - validateDefinitionFile(name, originalBranchesLatestFiles,) - } - - where: - branchesParam | latestFilesModified | name - ["9.2"] | ["9.1"] | "test_tv" // TODO legitimate bug, need to always clean up latest. - ["9.2"] | ["9.1"] | null // TODO legitimate bug, need to always clean up latest. - ["9.2", "9.1"] | ["9.2", "9.1"] | "test_tv" + def "generation fails if branches omitted outside CI"() { + when: + def generateResult = runGenerateTask("--name=no_branches").buildAndFail() + def validateResult = runValidateTask().build() + then: + assertGenerateFailure(generateResult) + assertOutputContains(generateResult.output, "When running outside CI, --branches must be specified") + assertValidateSuccess(validateResult) } - def "irrelevant changes to the latest file should be reverted"() { + def "invalid changes to a latest file should be reverted"() { given: latestTransportVersion("9.2", "modification", "9000000") when: - def result = runGenerateAndValidateTask("--branches=9.2") + def result = runGenerateAndValidateTask("--branches=9.2").build() then: assertGenerateAndValidateSuccess(result) assertLatest("9.2", "existing_92,8123000") } - def "irrelevant changes to multiple latest files should be reverted"() { + def "invalid changes to multiple latest files should be reverted"() { given: latestTransportVersion("9.2", "modification", "9000000") latestTransportVersion("9.1", "modification", "9000000") when: - def result = runGenerateAndValidateTask("--branches=9.2,9.1") + def result = runGenerateAndValidateTask("--branches=9.2,9.1").build() then: assertGenerateAndValidateSuccess(result) @@ -182,32 +125,32 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertLatest("9.1", "existing_92,8012001") } - def "state should be reverted when a reference is removed"() { + def "unreferenced referable definition should be reverted"() { given: namedTransportVersion("test_tv", "8124000") latestTransportVersion("9.2", "test_tv", "8124000") when: - def result = runGenerateAndValidateTask("--branches=9.2") + def result = runGenerateAndValidateTask().build() then: assertGenerateAndValidateSuccess(result) - !file("myserver/src/main/resources/transport/definitions/named/test_tv.csv").exists() + assertNamedDefinitionDoesNotExist("test_tv") assertLatest("9.2", "existing_92,8123000") } - def "state on multiple latest files should be reverted when a reference is removed"() { + def "unreferenced referable definition in multiple branches should be reverted"() { given: namedTransportVersion("test_tv", "8124000") latestTransportVersion("9.2", "test_tv", "8124000") latestTransportVersion("9.1", "test_tv", "8012002") when: - def result = runGenerateAndValidateTask("--branches=9.2") + def result = runGenerateAndValidateTask().build() then: assertGenerateAndValidateSuccess(result) - !file("myserver/src/main/resources/transport/definitions/named/test_tv.csv").exists() + assertNamedDefinitionDoesNotExist("test_tv") assertLatest("9.2", "existing_92,8123000") assertLatest("9.1", "existing_92,8012001") } @@ -219,234 +162,108 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes referencedTransportVersion("renamed_tv") when: - def result = runGenerateAndValidateTask("--branches=9.2") + def result = runGenerateAndValidateTask("--branches=9.2").build() then: assertGenerateAndValidateSuccess(result) - !file("myserver/src/main/resources/transport/definitions/named/first_tv.csv").exists() - file("myserver/src/main/resources/transport/definitions/named/renamed_tv.csv").exists() + assertNamedDefinitionDoesNotExist("first_tv") + assertNamedDefinition("renamed_tv", "8124000") assertLatest("9.2", "renamed_tv,8124000") } def "a reference with a patch version can be renamed"() { given: - namedTransportVersion("first_tv", "8124000") + namedTransportVersion("first_tv", "8124000,8012002") latestTransportVersion("9.2", "first_tv", "8124000") latestTransportVersion("9.1", "first_tv", "8012002") referencedTransportVersion("renamed_tv") when: - def result = runGenerateAndValidateTask("--branches=9.2,9.1") + def result = runGenerateAndValidateTask("--branches=9.2,9.1").build() then: assertGenerateAndValidateSuccess(result) - !file("myserver/src/main/resources/transport/definitions/named/first_tv.csv").exists() - file("myserver/src/main/resources/transport/definitions/named/renamed_tv.csv").exists() + assertNamedDefinitionDoesNotExist("first_tv") + assertNamedDefinition("renamed_tv", "8124000,8012002") assertLatest("9.2", "renamed_tv,8124000") assertLatest("9.1", "renamed_tv,8012002") } - def "The system should regenerate a deleted definition file"(List branches) { + def "a missing definition will be regenerated"() { given: - String definitionName = "test_tv" - referencedTransportVersion(definitionName) - List originalLatestFiles = readLatestFiles(branches) - - when: "The definition is generated" - def result = gradleRunner( - ":myserver:validateTransportVersionDefinitions", - ":myserver:generateTransportVersionDefinition", - "--name=" + definitionName, - "--branches=" + branches.join(",") - ).build() - - then: "The generation task should succeed and create the definition file" - result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS - result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS - validateDefinitionFile(definitionName, originalLatestFiles) - def latestFilesAfterGeneration = readLatestFiles(branches) - - when: "The definition file is deleted" - file("myserver/src/main/resources/transport/definitions/named/${definitionName}.csv").delete() - - then: "The definition file should no longer exist" - !file("myserver/src/main/resources/transport/definitions/named/${definitionName}.csv").exists() - - when: "Validation is run" - def validationResult = gradleRunner(":myserver:validateTransportVersionDefinitions").buildAndFail() - - then: "The validation task should fail since the definition file is missing" - validationResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.FAILED - - when: "The generation task is run again" - def secondResult = gradleRunner( - ":myserver:validateTransportVersionDefinitions", - ":myserver:generateTransportVersionDefinition", - "--branches=" + branches.join(",") - ).build() - - then: "The generation task should succeed and the definition file should be recreated" - secondResult.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS - secondResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS - validateDefinitionFile(definitionName, originalLatestFiles) - - and: "The latest files should remain unchanged" - latestFilesAfterGeneration.forEach { originalLatest -> - def latest = readLatestFile(originalLatest.branch) - assert latest.branch == originalLatest.branch - assert latest.id == originalLatest.id - assert latest.name == originalLatest.name - } - - where: - branches << [ - ["9.2"], - ["9.2", "9.1"] - ] + referencedTransportVersion("test_tv") + latestTransportVersion("9.2", "test_tv", "8124000") + + when: + def result = runGenerateAndValidateTask("--branches=9.2").build() + + then: + assertGenerateAndValidateSuccess(result) + assertNamedDefinition("test_tv", "8124000") + assertLatest("9.2", "test_tv,8124000") } - def "when a latest file is incorrectly changed and a referenced definition file exists, the latest file should be regenerated"(List branches) { + def "a latest file can be regenerated"() { given: - String definitionName = "test_tv" - referencedTransportVersion(definitionName) - List originalLatestFiles = readLatestFiles(branches) - - when: "The definition is generated" - def result = gradleRunner( - ":myserver:validateTransportVersionDefinitions", - ":myserver:generateTransportVersionDefinition", - "--name=" + definitionName, - "--branches=" + branches.join(",") - ).build() - - then: "The generation task should succeed and create the definition file" - result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS - result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS - validateDefinitionFile(definitionName, originalLatestFiles) - - when: "The latest files are modified" - List latestFilesAfterGeneration = readLatestFiles(branches) - originalLatestFiles.forEach { - latestTransportVersion(it.branch, it.name + "_modification", (it.id + 7).toString()) - } - - and: "The generation task is run again" - def secondResult = gradleRunner( - ":myserver:validateTransportVersionDefinitions", - ":myserver:generateTransportVersionDefinition", - "--branches=" + branches.join(",") - ).build() - - then: "The generation task should succeed and the latest files should be reverted and incremented correctly" - secondResult.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS - secondResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS - validateDefinitionFile(definitionName, originalLatestFiles) - - latestFilesAfterGeneration.forEach { originalLatest -> - def latest = readLatestFile(originalLatest.branch) - assert latest.branch == originalLatest.branch - assert latest.id == originalLatest.id - assert latest.name == originalLatest.name - } - - where: - branches << [ - ["9.2"], - ["9.2", "9.1"] - ] + definedAndUsedTransportVersion("test_tv", "8124000") + + when: + def result = runGenerateAndValidateTask("--branches=9.2").build() + + then: + assertGenerateAndValidateSuccess(result) + assertLatest("9.2", "test_tv,8124000") } - def "when a definition is created with a patch version, then generation is called without the patch version, the latest patch file should be reverted"() { + def "branches for definition can be removed"() { given: - String definitionName = "test_tv" - referencedTransportVersion(definitionName) - List branches = ["9.2", "9.1"] - List mainBranch = ["9.2"] - List originalLatestFilesWithPatch = readLatestFiles(branches) - List originalLatestMainFile = readLatestFiles(mainBranch) - LatestFile originalLatestPatchFile = readLatestFile("9.1") - - when: "The definition is generated with a patch version" - def result = gradleRunner( - ":myserver:validateTransportVersionDefinitions", - ":myserver:generateTransportVersionDefinition", - "--name=" + definitionName, - "--branches=" + branches.join(",") - ).build() - - then: "The generation task should succeed and create the definition file" - result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS - result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS - validateDefinitionFile(definitionName, originalLatestFilesWithPatch) - - when: "The generation is called again without the patch version" - def secondResult = gradleRunner( - ":myserver:validateTransportVersionDefinitions", - ":myserver:generateTransportVersionDefinition", - "--branches=" + mainBranch.join(",") - ).build() - - then: "The generation task should succeed and the definition file should be updated" - secondResult.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS - secondResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS - validateDefinitionFile(definitionName, originalLatestMainFile) - - and: "The latest file for the patch version should be reverted to the original state" - LatestFile latestPatchFile = readLatestFile("9.1") - latestPatchFile.id == originalLatestPatchFile.id - latestPatchFile.name == originalLatestPatchFile.name - latestPatchFile.branch == originalLatestPatchFile.branch + // previously generated with 9.1 and 9.2 + definedAndUsedTransportVersion("test_tv", "8124000,8012002") + latestTransportVersion("9.2", "test_tv", "8124000") + latestTransportVersion("9.1", "test_tv", "8012002") + + when: + def result = runGenerateAndValidateTask("--branches=9.2").build() + + then: + assertGenerateAndValidateSuccess(result) + assertNamedDefinition("test_tv", "8124000") + assertLatest("9.2", "test_tv,8124000") + assertLatest("9.1", "existing_92,8012001") } + def "branches for definition can be added"() { + given: + definedAndUsedTransportVersion("test_tv", "8124000") + latestTransportVersion("9.2", "test_tv", "8124000") + + when: + def result = runGenerateAndValidateTask("--branches=9.2,9.1").build() - def "when a reference is deleted, the system should delete the definition and revert the latest files"(List branches) { + then: + assertGenerateAndValidateSuccess(result) + assertNamedDefinition("test_tv", "8124000,8012002") + assertLatest("9.2", "test_tv,8124000") + assertLatest("9.1", "test_tv,8012002") + } + + def "unreferenced definitions are removed"() { given: - String definitionName = "test_tv" - referencedTransportVersion(definitionName) - List originalLatestFiles = readLatestFiles(branches) - - when: "The definition is generated" - def result = gradleRunner( - ":myserver:validateTransportVersionDefinitions", - ":myserver:generateTransportVersionDefinition", - "--name=" + definitionName, - "--branches=" + branches.join(",") - ).build() - - then: "The generation task should succeed and create the definition file" - result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS - result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS - validateDefinitionFile(definitionName, originalLatestFiles) - - when: "The reference is deleted and the generation is run again" - deleteTransportVersionReference(definitionName) - def secondResult = gradleRunner( - ":myserver:validateTransportVersionDefinitions", - ":myserver:generateTransportVersionDefinition", - "--branches=" + branches.join(",") - ).build() - - then: "The generation task should succeed and the definition file should be deleted" - !file("myserver/src/main/resources/transport/definitions/named/${definitionName}.csv").exists() - secondResult.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS - secondResult.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS - - and: "The latest files should be reverted to their original state" - originalLatestFiles.forEach { originalLatest -> - def latest = readLatestFile(originalLatest.branch) - assert latest.branch == originalLatest.branch - assert latest.id == originalLatest.id - assert latest.name == originalLatest.name - } - - where: - branches << [ - ["9.2"], - ["9.2", "9.1"] - ] + namedTransportVersion("test_tv", "8124000,8012002") + latestTransportVersion("9.2", "test_tv", "8124000") + latestTransportVersion("9.1", "test_tv", "8012002") + + when: + def result = runGenerateAndValidateTask().build() + + then: + assertGenerateAndValidateSuccess(result) + assertNamedDefinitionDoesNotExist("test_tv") + assertLatest("9.2", "existing_92,8123000") + assertLatest("9.1", "existing_92,8012001") } - def "latest files mangled by a merge conflict should be regenerated, and the most recent definition file should be updated"() { + def "merge conflicts in latest files can be regenerated"() { given: file("myserver/src/main/resources/transport/latest/9.2.csv").text = """ @@ -456,94 +273,15 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes second_tv,8123000 >>>>>> branch """.strip() - definedAndUsedTransportVersion("second_tv", "8123000") when: - def result = gradleRunner( - ":myserver:validateTransportVersionDefinitions", - ":myserver:generateTransportVersionDefinition", - "--name=second_tv", - "--branches=9.2" - ).build() - - then: "The generation task should succeed and the latest files should be reverted and incremented correctly" - result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS - result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS - file("myserver/src/main/resources/transport/definitions/named/existing_92.csv").text.strip() == "8123000,8012001" - file("myserver/src/main/resources/transport/definitions/named/second_tv.csv").text.strip() == "8124000" - file("myserver/src/main/resources/transport/latest/9.2.csv").text.strip() == "second_tv,8124000" - } + def result = runGenerateAndValidateTask("--name=second_tv", "--branches=9.2").build() - // TODO do we need a test that has a garbled latest file but Main is OK, and it increments correctly? - - List readLatestFiles(List branches) { - return branches.stream() - .map { readLatestFile(it) } - .toList() - } - - LatestFile readLatestFile(String branch) { - String latestFileText = file( - "myserver/src/main/resources/transport/latest/${branch}.csv" - ).text.strip() - assert latestFileText.isEmpty() == false: "The latest file must not be empty" - List parts = latestFileText.split(",") - assert parts.size() == 2: "The latest file must contain exactly two parts" - return new LatestFile(branch, parts[0], Integer.valueOf(parts[1])) - } - - void validateDefinitionFile(String definitionName, List originalLatestFiles, Integer primaryIncrement = 1000) { - String filename = "myserver/src/main/resources/transport/definitions/named/" + definitionName + ".csv" - assert file(filename).exists() - - String definitionFileText = file(filename).text.strip() - assert definitionFileText.isEmpty() == false: "The definition file must not be empty" - List definitionIDs = Arrays.stream(definitionFileText.split(",")).map(Integer::valueOf).toList() - assert originalLatestFiles.size() == definitionIDs.size(): "The definition file does not have an id for each latest file" - - def latestNamesToIds = originalLatestFiles.stream() - .map { it.branch } - .map { file("myserver/src/main/resources/transport/latest/${it}.csv").text } - .map { it.strip().split(",") } - .map { new Tuple2(it.first(), Integer.valueOf(it.last())) } - .toList() - - - for (int i = 0; i < definitionIDs.size(); i++) { - int definitionID = definitionIDs[i] - String nameInLatest = latestNamesToIds[i].getV1() - def idInLatest = latestNamesToIds[i].getV2() - - assert definitionName.equals(nameInLatest): "The latest and definition names must match" - assert definitionID == idInLatest: "The latest and definition ids must match" - - int originalID = originalLatestFiles[i].id - if (i == 0) { - assert definitionID == originalID + primaryIncrement: - "The primary version ID should be incremented by ${primaryIncrement} from the main branch latest file" - } else { - assert definitionID == originalID + 1: - "The patch version ID should be incremented by 1 from the primary version latest file" - } - - } - } - - class LatestFile { - String branch - String name - Integer id - - LatestFile(String branch, String name, Integer id) { - this.branch = branch - this.name = name - this.id = id - } - - @Override - String toString() { - return "LatestFile(branch=${branch}, name=${name}, id=${id})" - } + then: + assertGenerateAndValidateSuccess(result) + assertNamedDefinition("existing_92", "8123000,8012001") + assertNamedDefinition("second_tv", "8124000") + assertLatest("9.2", "second_tv,8124000") } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index 182e1c3379a5a..a3b8bced984ca 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -149,7 +149,7 @@ private List updateLatestFiles(TransportVersionResourcesServ } private Set getTargetReleaseBranches() { - if (getBranches().get().isEmpty() == false) { + if (getBranches().isPresent()) { return Arrays.stream(getBranches().get().split(",")) .map(branch -> branch.equals("main") ? getMainReleaseBranch().get() : branch) .collect(Collectors.toSet()); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java index b65debe84cc22..25ff2639ff594 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java @@ -10,6 +10,8 @@ package org.elasticsearch.gradle.internal.transport; import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; import org.gradle.api.services.BuildService; import org.gradle.api.services.BuildServiceParameters; import org.gradle.process.ExecOperations; @@ -51,6 +53,8 @@ */ public abstract class TransportVersionResourcesService implements BuildService { + private static final Logger logger = Logging.getLogger(TransportVersionResourcesService.class); + public interface Parameters extends BuildServiceParameters { DirectoryProperty getTransportResourcesDirectory(); @@ -147,8 +151,8 @@ Path getRepositoryPath(TransportVersionDefinition definition) { } void writeNamedDefinition(TransportVersionDefinition definition) throws IOException { - System.out.println("Writing definition: " + definition); Path path = transportResourcesDir.resolve(getNamedDefinitionRelativePath(definition.name())); + logger.debug("Writing referable definition [" + definition + "] to [" + path + "]"); Files.writeString( path, definition.ids().stream().map(Object::toString).collect(Collectors.joining(",")) + "\n", @@ -199,8 +203,8 @@ Path getRepositoryPath(TransportVersionLatest latest) { } void writeLatestFile(TransportVersionLatest latest) throws IOException { - System.out.println("Writing latest file: " + latest); Path path = transportResourcesDir.resolve(getLatestRelativePath(latest.releaseBranch())); + logger.debug("Writing upper bound [" + latest + "] to [" + path + "]"); Files.writeString(path, latest.name() + "," + latest.id().complete() + "\n", StandardCharsets.UTF_8); } From 12b01db2b1b7d5002ad6691ed3fe98954010965b Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Tue, 26 Aug 2025 12:53:30 -0700 Subject: [PATCH 37/68] more tests --- .../TransportVersionGenerationFuncTest.groovy | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index b3265716b3f67..fccfe4538ed70 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -284,4 +284,48 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertReferableDefinition("second_tv", "8124000") assertUpperBound("9.2", "second_tv,8124000") } + + def "branches param order can be changed"() { + given: + referableAndReferencedTransportVersion("test_tv", "8124000,8012002") + transportVersionUpperBound("9.2", "test_tv", "8124000") + transportVersionUpperBound("9.1", "test_tv", "8012002") + + when: + def result = runGenerateAndValidateTask("--branches=9.1,9.2").build() + + then: + assertGenerateAndValidateSuccess(result) + assertReferableDefinition("test_tv", "8124000,8012002") + assertUpperBound("9.2", "test_tv,8124000") + assertUpperBound("9.1", "test_tv,8012002") + } + + def "if the files for a new definition already exist, no change should occur"() { + given: + transportVersionUpperBound("9.2", "test_tv", "8124000") + referableAndReferencedTransportVersion("test_tv", "8124000") + + + when: + def result = runGenerateAndValidateTask("--name=test_tv", "--branches=9.2").build() + + then: + assertGenerateAndValidateSuccess(result) + assertReferableDefinition("test_tv", "8124000") + assertUpperBound("9.2", "test_tv,8124000") + } + + def "if the files for a committed definition already exist, no change should occur"() { + when: + def result = runGenerateAndValidateTask("--name=existing_92", "--branches=9.2").build() + + then: + assertGenerateAndValidateSuccess(result) + assertReferableDefinition("existing_92", "8123000,8012001") + assertUpperBound("9.2", "existing_92,8123000") + assertUpperBound("9.1", "existing_92,8012001") + } + + } From e0732f3c42059db96b7862781f825fa0d09febd9 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Tue, 26 Aug 2025 13:41:57 -0700 Subject: [PATCH 38/68] more tests --- .../TransportVersionGenerationFuncTest.groovy | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index fccfe4538ed70..e97d6097f30fd 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -308,7 +308,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes when: - def result = runGenerateAndValidateTask("--name=test_tv", "--branches=9.2").build() + def result = runGenerateAndValidateTask("--branches=9.2").build() then: assertGenerateAndValidateSuccess(result) @@ -328,4 +328,37 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes } + def "a different increment can be specified"() { + given: + referencedTransportVersion("new_tv") + + when: + def result = runGenerateAndValidateTask("--branches=9.2", "--increment=100").build() + + then: + assertGenerateAndValidateSuccess(result) + assertReferableDefinition("new_tv", "8124100") + assertUpperBound("9.2", "new_tv,8124100") + assertUpperBound("9.1", "existing_92,8012001") + } + + def "an invalid increment should fail"() { + // TODO should we have this? + } + + def "a new definition exists and is in the latest file, but the version id is wrong and needs to be updated"(){ + given: + referableAndReferencedTransportVersion("existing_92", "1000000") + transportVersionUpperBound("9.2", "existing_92", "1000000") + + when: + def result = runGenerateAndValidateTask("--branches=9.2").build() + + then: + assertGenerateAndValidateSuccess(result) + assertReferableDefinition("existing_92", "8123000,8012001") + assertUpperBound("9.2", "existing_92,8123000") + assertUpperBound("9.1", "existing_92,8012001") + } + } From 65c6a6ff8e5b49e8a25c700735caee5e1e2b976e Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 26 Aug 2025 14:11:32 -0700 Subject: [PATCH 39/68] fix stuff --- .../AbstractTransportVersionFuncTest.groovy | 1 + .../TransportVersionGenerationFuncTest.groovy | 7 ++--- ...enerateTransportVersionDefinitionTask.java | 27 +++++++++++++++---- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy index dc26883897cc6..eca7816353212 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy @@ -115,6 +115,7 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest { unreferableTransportVersion("initial_9_0_0", "8000000") transportVersionUpperBound("9.2", "existing_92", "8123000") transportVersionUpperBound("9.1", "existing_92", "8012001") + transportVersionUpperBound("9.0", "initial_9_0_0", "8000000") // a mock version of TransportVersion, just here so we can compile Dummy.java et al javaSource("myserver", "org.elasticsearch", "TransportVersion", "", """ public static TransportVersion fromName(String name) { diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index e97d6097f30fd..fd8045e1c9807 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -316,15 +316,16 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertUpperBound("9.2", "test_tv,8124000") } - def "if the files for a committed definition already exist, no change should occur"() { + def "can add backport to existing definition"() { when: - def result = runGenerateAndValidateTask("--name=existing_92", "--branches=9.2").build() + def result = runGenerateAndValidateTask("--name=existing_92", "--branches=9.2,9.1,9.0").build() then: assertGenerateAndValidateSuccess(result) - assertReferableDefinition("existing_92", "8123000,8012001") + assertReferableDefinition("existing_92", "8123000,8012001,8000001") assertUpperBound("9.2", "existing_92,8123000") assertUpperBound("9.1", "existing_92,8012001") + assertUpperBound("9.0", "existing_92,8000001") } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index 3748b26025c2d..0e2873ffb91fb 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -123,21 +123,38 @@ private List updateLatestFiles(TransportVersionResourcesServ int primaryIncrement = getPrimaryIncrement().get(); List ids = new ArrayList<>(); - for (TransportVersionUpperBound upperBound : resources.getUpperBoundsFromMain()) { + TransportVersionDefinition existingDefinition = resources.getReferableDefinitionFromMain(name); + for (TransportVersionUpperBound upperBound : resources.getUpperBoundsFromMain()) { + System.out.println("Looking at release branch: " + upperBound.releaseBranch()); if (name.equals(upperBound.name())) { if (targetReleaseBranches.contains(upperBound.releaseBranch()) == false) { // we don't want to target this latest file but we already changed it // Regenerate to make this operation idempotent. Need to undo prior updates to the latest files if the list of minor // versions has changed. resources.writeUpperBound(upperBound); + } else { + ids.add(upperBound.id()); } } else { if (targetReleaseBranches.contains(upperBound.releaseBranch())) { - int increment = upperBound.releaseBranch().equals(mainReleaseBranch) ? primaryIncrement : 1; - TransportVersionId id = TransportVersionId.fromInt(upperBound.id().complete() + increment); - ids.add(id); - resources.writeUpperBound(new TransportVersionUpperBound(upperBound.releaseBranch(), name, id)); + TransportVersionId targetId = null; + if (existingDefinition != null) { + System.out.println("Looking for upper bound base id: " + upperBound.id().base()); + for (TransportVersionId id : existingDefinition.ids()) { + System.out.println("Checking id: " + id); + if (id.base() == upperBound.id().base()) { + targetId = id; + break; + } + } + } + if (targetId == null) { + int increment = upperBound.releaseBranch().equals(mainReleaseBranch) ? primaryIncrement : 1; + TransportVersionId id = TransportVersionId.fromInt(upperBound.id().complete() + increment); + ids.add(id); + resources.writeUpperBound(new TransportVersionUpperBound(upperBound.releaseBranch(), name, id)); + } } else { resources.writeUpperBound(upperBound); } From ab026d912973edb8d4ffe6329d84b660d32f35f3 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Tue, 26 Aug 2025 14:54:36 -0700 Subject: [PATCH 40/68] more tests --- .../TransportVersionGenerationFuncTest.groovy | 23 +++++++++++++++++-- ...enerateTransportVersionDefinitionTask.java | 3 +++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index fd8045e1c9807..73a03ffafb395 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -316,7 +316,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertUpperBound("9.2", "test_tv,8124000") } - def "can add backport to existing definition"() { + def "can add backport to an existing definition"() { when: def result = runGenerateAndValidateTask("--name=existing_92", "--branches=9.2,9.1,9.0").build() @@ -344,7 +344,14 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes } def "an invalid increment should fail"() { - // TODO should we have this? + given: + referencedTransportVersion("new_tv") + + when: + def result = runGenerateTask("--branches=9.2", "--increment=0").buildAndFail() + + then: + assertOutputContains(result.output, "Invalid increment: must be a positive integer > 0") } def "a new definition exists and is in the latest file, but the version id is wrong and needs to be updated"(){ @@ -362,4 +369,16 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertUpperBound("9.1", "existing_92,8012001") } + def "branches=main should be translated to 9.2"() { + given: + referencedTransportVersion("new_tv") + + when: + def result = runGenerateAndValidateTask("--branches=main").build() + + then: + assertGenerateAndValidateSuccess(result) + assertReferableDefinition("new_tv", "8124000") + assertUpperBound("9.2", "new_tv,8124000") + } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index 0e2873ffb91fb..a6a0c54131151 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -121,6 +121,9 @@ private List updateLatestFiles(TransportVersionResourcesServ Set targetReleaseBranches = getTargetReleaseBranches(); String mainReleaseBranch = getMainReleaseBranch().get(); int primaryIncrement = getPrimaryIncrement().get(); + if (primaryIncrement <= 0) { + throw new IllegalArgumentException("Invalid increment: must be a positive integer > 0"); + } List ids = new ArrayList<>(); TransportVersionDefinition existingDefinition = resources.getReferableDefinitionFromMain(name); From 37a5707c6701dd197f6c56bd324975323dcd7cc9 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 26 Aug 2025 16:51:24 -0700 Subject: [PATCH 41/68] fix increment --- .../TransportVersionGenerationFuncTest.groovy | 10 +++++++--- .../transport/TransportVersionResourcesPlugin.java | 1 + .../ValidateTransportVersionResourcesTask.java | 6 +++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index 73a03ffafb395..c9fc4fd58b56c 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -332,15 +332,19 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes def "a different increment can be specified"() { given: referencedTransportVersion("new_tv") + file("myserver/build.gradle") << """ + tasks.named('validateTransportVersionResources') { + shouldValidateDensity = false + } + """ when: def result = runGenerateAndValidateTask("--branches=9.2", "--increment=100").build() then: assertGenerateAndValidateSuccess(result) - assertReferableDefinition("new_tv", "8124100") - assertUpperBound("9.2", "new_tv,8124100") - assertUpperBound("9.1", "existing_92,8012001") + assertReferableDefinition("new_tv", "8123100") + assertUpperBound("9.2", "new_tv,8123100") } def "an invalid increment should fail"() { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java index 238974e2543bc..0c5490b1d7cc8 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java @@ -56,6 +56,7 @@ public void apply(Project project) { t.setGroup("Transport Versions"); t.setDescription("Validates that all transport version resources are internally consistent with each other"); t.getReferencesFiles().setFrom(tvReferencesConfig); + t.getShouldValidateDensity().convention(true); }); project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(validateTask)); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java index 21a75dc401040..2d489a7a3c1ee 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java @@ -16,6 +16,7 @@ import org.gradle.api.provider.Property; import org.gradle.api.services.ServiceReference; import org.gradle.api.tasks.CacheableTask; +import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputDirectory; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.Optional; @@ -51,6 +52,9 @@ public Path getResourcesDir() { @PathSensitive(PathSensitivity.RELATIVE) public abstract ConfigurableFileCollection getReferencesFiles(); + @Input + public abstract Property getShouldValidateDensity(); + private record IdAndDefinition(TransportVersionId id, TransportVersionDefinition definition) {} private static final Pattern NAME_FORMAT = Pattern.compile("[a-z0-9_]+"); @@ -244,7 +248,7 @@ private void validateBase(int base, List ids) { ); } - if (previous.id().complete() - 1 != current.id().complete()) { + if (getShouldValidateDensity().get() && previous.id().complete() - 1 != current.id().complete()) { throw new IllegalStateException( "Transport version base id " + base + " is missing patch ids between " + current.id() + " and " + previous.id() ); From ac14cd2d76464e6f07e6a71a5b5dde2b7c9f61b5 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Wed, 27 Aug 2025 07:35:14 -0700 Subject: [PATCH 42/68] fix test --- .../transport/TransportVersionGenerationFuncTest.groovy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index c9fc4fd58b56c..dbb01dd6ec985 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -9,6 +9,8 @@ package org.elasticsearch.gradle.internal.transport +import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.VersionProperties import org.gradle.testkit.runner.BuildResult import org.gradle.testkit.runner.TaskOutcome @@ -373,9 +375,11 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertUpperBound("9.1", "existing_92,8012001") } - def "branches=main should be translated to 9.2"() { + def "branches=main should be translated"() { given: referencedTransportVersion("new_tv") + Version esVersion = VersionProperties.getElasticsearchVersion(); + String latestVersion = esVersion.getMajor() + "." + esVersion.getMinor() when: def result = runGenerateAndValidateTask("--branches=main").build() @@ -383,6 +387,6 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes then: assertGenerateAndValidateSuccess(result) assertReferableDefinition("new_tv", "8124000") - assertUpperBound("9.2", "new_tv,8124000") + assertUpperBound(latestVersion, "new_tv,8124000") } } From ac9a56d8f8c3f00b29f039874041e62b36343222 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 27 Aug 2025 11:58:35 -0700 Subject: [PATCH 43/68] fix main test --- .../AbstractTransportVersionFuncTest.groovy | 4 ++++ .../TransportVersionGenerationFuncTest.groovy | 22 +++++++++++++++---- .../TransportVersionResourcesPlugin.java | 2 +- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy index eca7816353212..21adf6ba77456 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy @@ -109,6 +109,10 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest { apply plugin: 'java-library' apply plugin: 'elasticsearch.transport-version-references' apply plugin: 'elasticsearch.transport-version-resources' + + tasks.named('generateTransportVersionDefinition') { + mainReleaseBranch = '9.2' + } """ referableTransportVersion("existing_91", "8012000") referableTransportVersion("existing_92", "8123000,8012001") diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index dbb01dd6ec985..a088bd4a30c05 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -52,6 +52,10 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertValidateSuccess(result) } + def setup() { + + } + def "setup is valid"() { when: def result = runGenerateAndValidateTask().build() @@ -377,16 +381,26 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes def "branches=main should be translated"() { given: + execute("git checkout main") + Version esVersion = VersionProperties.getElasticsearchVersion() + String releaseBranch = esVersion.getMajor() + "." + esVersion.getMinor() + unreferableTransportVersion("initial_main", "9000000") + transportVersionUpperBound(releaseBranch, "initial_main", "9000000") + file('myserver/build.gradle') << """ + tasks.named('generateTransportVersionDefinition') { + getMainReleaseBranch().unset() + } + """ + execute('git commit -a -m "setup_main"') + execute('git checkout -b new_branch') referencedTransportVersion("new_tv") - Version esVersion = VersionProperties.getElasticsearchVersion(); - String latestVersion = esVersion.getMajor() + "." + esVersion.getMinor() when: def result = runGenerateAndValidateTask("--branches=main").build() then: assertGenerateAndValidateSuccess(result) - assertReferableDefinition("new_tv", "8124000") - assertUpperBound(latestVersion, "new_tv,8124000") + assertReferableDefinition("new_tv", "9001000") + assertUpperBound(releaseBranch, "new_tv,9001000") } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java index 0c5490b1d7cc8..06e4b2e5c52dc 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java @@ -77,7 +77,7 @@ public void apply(Project project) { t.getReferencesFiles().setFrom(tvReferencesConfig); t.getPrimaryIncrement().convention(1000); Version esVersion = VersionProperties.getElasticsearchVersion(); - t.getMainReleaseBranch().set(esVersion.getMajor() + "." + esVersion.getMinor()); + t.getMainReleaseBranch().convention(esVersion.getMajor() + "." + esVersion.getMinor()); t.getResourcesProjectDir() .set(project.getRootProject().getProjectDir().toPath().relativize(project.getProjectDir().toPath()).toString()); }); From da0837002dd94daeda39cfa7ab80c65424cad2f9 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Wed, 27 Aug 2025 13:33:11 -0700 Subject: [PATCH 44/68] fix test --- .../transport/TransportVersionGenerationFuncTest.groovy | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index a088bd4a30c05..9daa17b2d105f 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -366,17 +366,16 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes def "a new definition exists and is in the latest file, but the version id is wrong and needs to be updated"(){ given: - referableAndReferencedTransportVersion("existing_92", "1000000") - transportVersionUpperBound("9.2", "existing_92", "1000000") + referableAndReferencedTransportVersion("new_tv", "1000000") + transportVersionUpperBound("9.2", "new_tv", "1000000") when: def result = runGenerateAndValidateTask("--branches=9.2").build() then: assertGenerateAndValidateSuccess(result) - assertReferableDefinition("existing_92", "8123000,8012001") - assertUpperBound("9.2", "existing_92,8123000") - assertUpperBound("9.1", "existing_92,8012001") + assertReferableDefinition("new_tv", "8124000") + assertUpperBound("9.2", "new_tv,8124000") } def "branches=main should be translated"() { From d5c0cd5e23f22acc4c940a598d2a42c5af1c7542 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Wed, 27 Aug 2025 14:05:44 -0700 Subject: [PATCH 45/68] cleanup --- ...enerateTransportVersionDefinitionTask.java | 42 +++++++------------ .../TransportVersionResourcesPlugin.java | 2 - 2 files changed, 14 insertions(+), 30 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index a6a0c54131151..c7b46d4ef9aab 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -24,6 +24,7 @@ import org.gradle.process.ExecOperations; import org.gradle.process.ExecResult; +import javax.inject.Inject; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.file.Path; @@ -35,8 +36,6 @@ import java.util.Set; import java.util.stream.Collectors; -import javax.inject.Inject; - /** * This task generates transport version definition files. These files * are runtime resources that TransportVersion loads statically. @@ -59,23 +58,14 @@ public Path getResourcesDir() { @ServiceReference("transportVersionResources") abstract Property getResources(); - // assumption: this task is always run on main, so we can determine the name by diffing with main and looking for new files added in the - // definition directory. (not true: once we generate the file, this will no longer hold true if we then need to update it) - - /** - * Used to set the name of the TransportVersionSet for which a data file will be generated. - */ @Input @Optional - @Option(option = "name", description = "TBD") // TODO add description - public abstract Property getTransportVersionName(); // The plugin should always set this, not optional + @Option(option = "name", description = "The name of the Transport Version reference") + public abstract Property getTransportVersionName(); - /** - * The release releaseBranch names the generated transport version should target. - */ - @Optional // In CI we find these from the github PR labels + @Optional @Input - @Option(option = "branches", description = "The branches for which to generate IDs, e.g. --branches=\"main,9.1\"") + @Option(option = "branches", description = "The release branches for which to generate IDs, e.g. --branches=main,9.1") public abstract Property getBranches(); @Input @@ -83,12 +73,13 @@ public Path getResourcesDir() { @Option(option = "increment", description = "The amount to increment the primary id for the main releaseBranch") public abstract Property getPrimaryIncrement(); + /** + * The version number currently associated with the `main` branch, (e.g. as returned by + * VersionProperties.getElasticsearchVersion()). + */ @Input public abstract Property getMainReleaseBranch(); - @Input - public abstract Property getResourcesProjectDir(); - private final ExecOperations execOperations; @Inject @@ -106,7 +97,6 @@ public void run() throws IOException { : findAddedTransportVersionName(resources, referencedNames, changedDefinitionNames); if (name.isEmpty()) { - // Todo this should reset all the changed latest files regardless of if there's a name or not resetAllLatestFiles(resources); } else { List ids = updateLatestFiles(resources, name); @@ -129,12 +119,10 @@ private List updateLatestFiles(TransportVersionResourcesServ TransportVersionDefinition existingDefinition = resources.getReferableDefinitionFromMain(name); for (TransportVersionUpperBound upperBound : resources.getUpperBoundsFromMain()) { - System.out.println("Looking at release branch: " + upperBound.releaseBranch()); if (name.equals(upperBound.name())) { if (targetReleaseBranches.contains(upperBound.releaseBranch()) == false) { - // we don't want to target this latest file but we already changed it - // Regenerate to make this operation idempotent. Need to undo prior updates to the latest files if the list of minor - // versions has changed. + // Here, we aren't targeting this latest file, but need to undo prior updates if the list of minor + // versions has changed. We must regenerate this latest file to make this operation idempotent. resources.writeUpperBound(upperBound); } else { ids.add(upperBound.id()); @@ -143,9 +131,7 @@ private List updateLatestFiles(TransportVersionResourcesServ if (targetReleaseBranches.contains(upperBound.releaseBranch())) { TransportVersionId targetId = null; if (existingDefinition != null) { - System.out.println("Looking for upper bound base id: " + upperBound.id().base()); for (TransportVersionId id : existingDefinition.ids()) { - System.out.println("Checking id: " + id); if (id.base() == upperBound.id().base()) { targetId = id; break; @@ -174,9 +160,9 @@ private Set getTargetReleaseBranches() { .map(branch -> branch.equals("main") ? getMainReleaseBranch().get() : branch) .collect(Collectors.toSet()); } else { - // look for env var indicating github PR link from CI - // use github api to find current labels, filter down to version labels - // map version labels to branches + // Look for env var indicating github PR link from CI. + // Use github api to find current labels, filter down to version labels. + // Map version labels to branches. String prUrl = System.getenv("BUILDKITE_PULL_REQUEST"); if (prUrl == null) { throw new RuntimeException("When running outside CI, --branches must be specified"); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java index 06e4b2e5c52dc..5366f20f53305 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java @@ -78,8 +78,6 @@ public void apply(Project project) { t.getPrimaryIncrement().convention(1000); Version esVersion = VersionProperties.getElasticsearchVersion(); t.getMainReleaseBranch().convention(esVersion.getMajor() + "." + esVersion.getMinor()); - t.getResourcesProjectDir() - .set(project.getRootProject().getProjectDir().toPath().relativize(project.getProjectDir().toPath()).toString()); }); validateTask.configure(t -> t.mustRunAfter(generateDefinitionsTask)); From 29276a73bc746ca1ce8698d29f1c92fcabe81d3e Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 27 Aug 2025 21:14:17 +0000 Subject: [PATCH 46/68] [CI] Auto commit changes from spotless --- .../transport/GenerateTransportVersionDefinitionTask.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index c7b46d4ef9aab..38e528fbdb748 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -24,7 +24,6 @@ import org.gradle.process.ExecOperations; import org.gradle.process.ExecResult; -import javax.inject.Inject; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.file.Path; @@ -36,6 +35,8 @@ import java.util.Set; import java.util.stream.Collectors; +import javax.inject.Inject; + /** * This task generates transport version definition files. These files * are runtime resources that TransportVersion loads statically. From 4259b589b4385750104f5d87e53fa7d1310bb200 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Thu, 28 Aug 2025 12:15:27 -0700 Subject: [PATCH 47/68] cleanup --- ...enerateTransportVersionDefinitionTask.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index 38e528fbdb748..ec419f6f350e5 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -14,7 +14,6 @@ import org.gradle.api.provider.Property; import org.gradle.api.services.ServiceReference; import org.gradle.api.tasks.Input; -import org.gradle.api.tasks.InputDirectory; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.Optional; import org.gradle.api.tasks.PathSensitive; @@ -24,9 +23,9 @@ import org.gradle.process.ExecOperations; import org.gradle.process.ExecResult; +import javax.inject.Inject; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -35,8 +34,6 @@ import java.util.Set; import java.util.stream.Collectors; -import javax.inject.Inject; - /** * This task generates transport version definition files. These files * are runtime resources that TransportVersion loads statically. @@ -45,28 +42,31 @@ */ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask { - @InputDirectory - @Optional - @PathSensitive(PathSensitivity.RELATIVE) - public Path getResourcesDir() { - return getResources().get().getTransportResourcesDir(); - } - + /** + * Files that contain the references to the transport version names. + */ @InputFiles @PathSensitive(PathSensitivity.RELATIVE) public abstract ConfigurableFileCollection getReferencesFiles(); @ServiceReference("transportVersionResources") - abstract Property getResources(); + abstract Property getResourceService(); @Input @Optional - @Option(option = "name", description = "The name of the Transport Version reference") + @Option( + option = "name", + description = "The name of the Transport Version reference for which to generate a definition, e.g. --name=my_new_tv" + ) public abstract Property getTransportVersionName(); @Optional @Input - @Option(option = "branches", description = "The release branches for which to generate IDs, e.g. --branches=main,9.1") + @Option( + option = "branches", + description = "The release branches for which to generate IDs, e.g. --branches=main,9.1. " + + "Must always include the main release branch, and backport branches must be contiguous from main." + ) public abstract Property getBranches(); @Input @@ -90,7 +90,7 @@ public GenerateTransportVersionDefinitionTask(ExecOperations execOperations) { @TaskAction public void run() throws IOException { - TransportVersionResourcesService resources = getResources().get(); + TransportVersionResourcesService resources = getResourceService().get(); Set referencedNames = TransportVersionReference.collectNames(getReferencesFiles()); List changedDefinitionNames = resources.getChangedReferableDefinitionNames(); String name = getTransportVersionName().isPresent() From 401372531de46c50561939079c588f34f66058e0 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Thu, 28 Aug 2025 12:19:04 -0700 Subject: [PATCH 48/68] cleanup --- .../internal/transport/TransportVersionResourcesService.java | 2 ++ .../gradle/internal/transport/TransportVersionUpperBound.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java index 3c70fed988308..6ba3b399383b3 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java @@ -190,6 +190,7 @@ TransportVersionUpperBound getUpperBoundFromMain(String releaseBranch) { return getMainFile(resourcePath, TransportVersionUpperBound::fromString); } + /** Retrieve all upper bounds that exist on main */ List getUpperBoundsFromMain() throws IOException { List upperBounds = new ArrayList<>(); for (String mainPathString : getMainResources()) { @@ -203,6 +204,7 @@ List getUpperBoundsFromMain() throws IOException { return upperBounds; } + /** Write the given upper bound to a file in the transport resources */ void writeUpperBound(TransportVersionUpperBound upperBound) throws IOException { Path path = transportResourcesDir.resolve(getUpperBoundRelativePath(upperBound.releaseBranch())); logger.debug("Writing upper bound [" + upperBound + "] to [" + path + "]"); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUpperBound.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUpperBound.java index 3d306a5f42795..f675d7db67a20 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUpperBound.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUpperBound.java @@ -14,7 +14,7 @@ /** * An object to represent the loaded version of a transport version upper bound. * - * An upper bound is the maximum transport version id that should be loaded for a given release releaseBranch. + * An upper bound is the maximum transport version id that should be loaded for a given release branch. */ record TransportVersionUpperBound(String releaseBranch, String name, TransportVersionId id) { public static TransportVersionUpperBound fromString(Path file, String contents) { From 6385574df76900b2ed59a01156519376fa35c415 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 28 Aug 2025 19:27:25 +0000 Subject: [PATCH 49/68] [CI] Auto commit changes from spotless --- .../transport/GenerateTransportVersionDefinitionTask.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index ec419f6f350e5..481c02e4e5b1b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -23,7 +23,6 @@ import org.gradle.process.ExecOperations; import org.gradle.process.ExecResult; -import javax.inject.Inject; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; @@ -34,6 +33,8 @@ import java.util.Set; import java.util.stream.Collectors; +import javax.inject.Inject; + /** * This task generates transport version definition files. These files * are runtime resources that TransportVersion loads statically. @@ -64,8 +65,8 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask @Input @Option( option = "branches", - description = "The release branches for which to generate IDs, e.g. --branches=main,9.1. " + - "Must always include the main release branch, and backport branches must be contiguous from main." + description = "The release branches for which to generate IDs, e.g. --branches=main,9.1. " + + "Must always include the main release branch, and backport branches must be contiguous from main." ) public abstract Property getBranches(); From b2dbfefe7d358054e02a4bfa891fc0d65349b042 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 28 Aug 2025 17:37:46 -0700 Subject: [PATCH 50/68] switch to explicit env var for labels --- .../TransportVersionGenerationFuncTest.groovy | 61 +++++++++++++++++++ ...enerateTransportVersionDefinitionTask.java | 16 +---- .../fixtures/AbstractGradleFuncTest.groovy | 2 + 3 files changed, 66 insertions(+), 13 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index 9daa17b2d105f..5bc32b4810c1e 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -11,9 +11,12 @@ package org.elasticsearch.gradle.internal.transport import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.VersionProperties +import org.elasticsearch.gradle.fixtures.WiremockFixture import org.gradle.testkit.runner.BuildResult import org.gradle.testkit.runner.TaskOutcome +import java.nio.charset.StandardCharsets + class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTest { def runGenerateAndValidateTask(String... additionalArgs) { @@ -24,6 +27,11 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes return gradleRunner(args.toArray()) } + def runGenerateAndValidateWithLabels(String... labels) { + String labelsStr = String.join(System.lineSeparator(), List.of(labels)); + return runGenerateAndValidateTask().withEnvironment(Map.of("PULL_REQUEST_LABELS", labelsStr)); + } + def runGenerateTask(String... additionalArgs) { List args = new ArrayList<>() args.add(":myserver:generateTransportVersionDefinition") @@ -402,4 +410,57 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertReferableDefinition("new_tv", "9001000") assertUpperBound(releaseBranch, "new_tv,9001000") } + + def "branch from pr targeting main"() { + given: + referencedTransportVersion("new_tv") + + when: + def result = runGenerateAndValidateWithLabels("v9.2.0").build() + + then: + assertGenerateAndValidateSuccess(result) + assertReferableDefinition("new_tv", "8124000") + assertUpperBound("9.2", "new_tv,8124000") + } + + def "branch from pr for backport"() { + given: + referencedTransportVersion("new_tv") + + when: + def result = runGenerateAndValidateWithLabels("v9.2.0", "v9.1.2").build() + + then: + assertGenerateAndValidateSuccess(result) + assertReferableDefinition("new_tv", "8124000,8012002") + assertUpperBound("9.2", "new_tv,8124000") + assertUpperBound("9.1", "new_tv,8012002") + } + + def "branch from pr with other labels"() { + given: + referencedTransportVersion("new_tv") + + when: + def result = runGenerateAndValidateWithLabels(">test", "v9.2.0", ":Core/Infra/Core").build() + + then: + assertGenerateAndValidateSuccess(result) + assertReferableDefinition("new_tv", "8124000") + assertUpperBound("9.2", "new_tv,8124000") + } + + def "branch from pr default to main"() { + given: + referencedTransportVersion("new_tv") + + when: + def result = runGenerateAndValidateWithLabels(">test", ":Core/Infra/Core").build() + + then: + assertGenerateAndValidateSuccess(result) + assertReferableDefinition("new_tv", "8124000") + assertUpperBound("9.2", "new_tv,8124000") + } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index 481c02e4e5b1b..bd343db9d9f0e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -165,23 +165,13 @@ private Set getTargetReleaseBranches() { // Look for env var indicating github PR link from CI. // Use github api to find current labels, filter down to version labels. // Map version labels to branches. - String prUrl = System.getenv("BUILDKITE_PULL_REQUEST"); - if (prUrl == null) { + String prLabels = System.getenv("PULL_REQUEST_LABELS"); + if (prLabels == null) { throw new RuntimeException("When running outside CI, --branches must be specified"); } - ByteArrayOutputStream output = new ByteArrayOutputStream(); - ExecResult result = execOperations.exec(spec -> { - spec.setCommandLine(List.of("gh", "pr", "view", prUrl, "--json", "labels", "--jq", ".labels[].name")); - spec.setErrorOutput(output); - spec.setStandardOutput(output); - spec.setIgnoreExitValue(true); - }); - if (result.getExitValue() != 0) { - throw new RuntimeException("Failed to get labels from github API:\n" + output); - } Set targetReleaseBranches = new HashSet<>(); - for (String label : output.toString().split(System.lineSeparator())) { + for (String label : prLabels.split(System.lineSeparator())) { if (label.startsWith("v") == false) { continue; } diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy index aa9b9d07c4461..a40921b19dea4 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy @@ -202,11 +202,13 @@ abstract class AbstractGradleFuncTest extends Specification { void execute(String command, File workingDir = testProjectDir.root) { def proc = command.execute(Collections.emptyList(), workingDir) + System.out.println(command) proc.waitFor() if (proc.exitValue()) { System.err.println("Error running command ${command}:") System.err.println("Syserr: " + proc.errorStream.text) } + System.out.println("Output: " + proc.inputStream.text) } File dir(String path) { From caa5a707840dd30f6eac0887eb45c8485f9fcb19 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 29 Aug 2025 00:45:28 +0000 Subject: [PATCH 51/68] [CI] Auto commit changes from spotless --- .../transport/GenerateTransportVersionDefinitionTask.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index bd343db9d9f0e..f20f5698a7d4e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -21,9 +21,7 @@ import org.gradle.api.tasks.TaskAction; import org.gradle.api.tasks.options.Option; import org.gradle.process.ExecOperations; -import org.gradle.process.ExecResult; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; From 9ade26c58f08482f062e8dae8dc74a124320a739 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 29 Aug 2025 10:20:52 -0700 Subject: [PATCH 52/68] iter --- .../TransportVersionGenerationFuncTest.groovy | 141 ++++++------------ ...enerateTransportVersionDefinitionTask.java | 111 +++++++------- .../fixtures/AbstractGradleFuncTest.groovy | 2 - 3 files changed, 97 insertions(+), 157 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index 5bc32b4810c1e..97c1331899494 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -27,11 +27,6 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes return gradleRunner(args.toArray()) } - def runGenerateAndValidateWithLabels(String... labels) { - String labelsStr = String.join(System.lineSeparator(), List.of(labels)); - return runGenerateAndValidateTask().withEnvironment(Map.of("PULL_REQUEST_LABELS", labelsStr)); - } - def runGenerateTask(String... additionalArgs) { List args = new ArrayList<>() args.add(":myserver:generateTransportVersionDefinition") @@ -47,8 +42,9 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assert result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.SUCCESS } - void assertGenerateFailure(BuildResult result) { + void assertGenerateFailure(BuildResult result, String expectedOutput) { assert result.task(":myserver:generateTransportVersionDefinition").outcome == TaskOutcome.FAILED + assertOutputContains(result.output, expectedOutput) } void assertValidateSuccess(BuildResult result) { @@ -60,10 +56,6 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertValidateSuccess(result) } - def setup() { - - } - def "setup is valid"() { when: def result = runGenerateAndValidateTask().build() @@ -74,7 +66,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes def "a definition should be generated when specified by an arg but no code reference exists yet"() { when: - def result = runGenerateTask("--name=new_tv", "--branches=9.2").build() + def result = runGenerateTask("--name=new_tv").build() then: assertGenerateSuccess(result) @@ -94,7 +86,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes referencedTransportVersion("new_tv") when: - def result = runGenerateAndValidateTask("--branches=9.2").build() + def result = runGenerateAndValidateTask().build() then: assertGenerateAndValidateSuccess(result) @@ -102,23 +94,12 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertUpperBound("9.2", "new_tv,8124000") } - def "generation fails if branches omitted outside CI"() { - when: - def generateResult = runGenerateTask("--name=no_branches").buildAndFail() - def validateResult = runValidateTask().build() - - then: - assertGenerateFailure(generateResult) - assertOutputContains(generateResult.output, "When running outside CI, --branches must be specified") - assertValidateSuccess(validateResult) - } - def "invalid changes to a latest file should be reverted"() { given: transportVersionUpperBound("9.2", "modification", "9000000") when: - def result = runGenerateAndValidateTask("--branches=9.2").build() + def result = runGenerateAndValidateTask().build() then: assertGenerateAndValidateSuccess(result) @@ -131,7 +112,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes transportVersionUpperBound("9.1", "modification", "9000000") when: - def result = runGenerateAndValidateTask("--branches=9.2,9.1").build() + def result = runGenerateAndValidateTask("--backport-branches=9.1").build() then: assertGenerateAndValidateSuccess(result) @@ -176,7 +157,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes referencedTransportVersion("renamed_tv") when: - def result = runGenerateAndValidateTask("--branches=9.2").build() + def result = runGenerateAndValidateTask().build() then: assertGenerateAndValidateSuccess(result) @@ -193,7 +174,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes referencedTransportVersion("renamed_tv") when: - def result = runGenerateAndValidateTask("--branches=9.2,9.1").build() + def result = runGenerateAndValidateTask("--backport-branches=9.1").build() then: assertGenerateAndValidateSuccess(result) @@ -209,7 +190,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes transportVersionUpperBound("9.2", "test_tv", "8124000") when: - def result = runGenerateAndValidateTask("--branches=9.2").build() + def result = runGenerateAndValidateTask().build() then: assertGenerateAndValidateSuccess(result) @@ -222,7 +203,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes referableAndReferencedTransportVersion("test_tv", "8124000") when: - def result = runGenerateAndValidateTask("--branches=9.2").build() + def result = runGenerateAndValidateTask().build() then: assertGenerateAndValidateSuccess(result) @@ -237,7 +218,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes transportVersionUpperBound("9.1", "test_tv", "8012002") when: - def result = runGenerateAndValidateTask("--branches=9.2").build() + def result = runGenerateAndValidateTask().build() then: assertGenerateAndValidateSuccess(result) @@ -252,7 +233,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes transportVersionUpperBound("9.2", "test_tv", "8124000") when: - def result = runGenerateAndValidateTask("--branches=9.2,9.1").build() + def result = runGenerateAndValidateTask("--backport-branches=9.1").build() then: assertGenerateAndValidateSuccess(result) @@ -290,7 +271,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes referableAndReferencedTransportVersion("second_tv", "8123000") when: - def result = runGenerateAndValidateTask("--name=second_tv", "--branches=9.2").build() + def result = runGenerateAndValidateTask("--name=second_tv", ).build() then: assertGenerateAndValidateSuccess(result) @@ -299,20 +280,19 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertUpperBound("9.2", "second_tv,8124000") } - def "branches param order can be changed"() { + def "branches param order does not matter"() { given: - referableAndReferencedTransportVersion("test_tv", "8124000,8012002") - transportVersionUpperBound("9.2", "test_tv", "8124000") - transportVersionUpperBound("9.1", "test_tv", "8012002") + referencedTransportVersion("test_tv") when: - def result = runGenerateAndValidateTask("--branches=9.1,9.2").build() + def result = runGenerateAndValidateTask("--backport-branches=9.0,9.1").build() then: assertGenerateAndValidateSuccess(result) - assertReferableDefinition("test_tv", "8124000,8012002") + assertReferableDefinition("test_tv", "8124000,8012002,8000001") assertUpperBound("9.2", "test_tv,8124000") assertUpperBound("9.1", "test_tv,8012002") + assertUpperBound("9.0", "test_tv,8000001") } def "if the files for a new definition already exist, no change should occur"() { @@ -322,7 +302,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes when: - def result = runGenerateAndValidateTask("--branches=9.2").build() + def result = runGenerateAndValidateTask().build() then: assertGenerateAndValidateSuccess(result) @@ -332,7 +312,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes def "can add backport to an existing definition"() { when: - def result = runGenerateAndValidateTask("--name=existing_92", "--branches=9.2,9.1,9.0").build() + def result = runGenerateAndValidateTask("--name=existing_92", "--backport-branches=9.1,9.0").build() then: assertGenerateAndValidateSuccess(result) @@ -342,6 +322,18 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertUpperBound("9.0", "existing_92,8000001") } + def "increment must be positive"() { + given: + referencedTransportVersion("new_tv") + + when: + def result = runGenerateTask("--increment=0").buildAndFail() + + then: + assertGenerateFailure(result, "Invalid increment [0], must be a positive integer") + assertReferableDefinitionDoesNotExist("new_tv") + assertUpperBound("9.2", "existing_92,8123000") + } def "a different increment can be specified"() { given: @@ -353,7 +345,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes """ when: - def result = runGenerateAndValidateTask("--branches=9.2", "--increment=100").build() + def result = runGenerateAndValidateTask("--backport-branches=9.2", "--increment=100").build() then: assertGenerateAndValidateSuccess(result) @@ -366,7 +358,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes referencedTransportVersion("new_tv") when: - def result = runGenerateTask("--branches=9.2", "--increment=0").buildAndFail() + def result = runGenerateTask("--increment=0").buildAndFail() then: assertOutputContains(result.output, "Invalid increment: must be a positive integer > 0") @@ -378,7 +370,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes transportVersionUpperBound("9.2", "new_tv", "1000000") when: - def result = runGenerateAndValidateTask("--branches=9.2").build() + def result = runGenerateAndValidateTask().build() then: assertGenerateAndValidateSuccess(result) @@ -386,37 +378,12 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertUpperBound("9.2", "new_tv,8124000") } - def "branches=main should be translated"() { + def "backport branches is optional"() { given: - execute("git checkout main") - Version esVersion = VersionProperties.getElasticsearchVersion() - String releaseBranch = esVersion.getMajor() + "." + esVersion.getMinor() - unreferableTransportVersion("initial_main", "9000000") - transportVersionUpperBound(releaseBranch, "initial_main", "9000000") - file('myserver/build.gradle') << """ - tasks.named('generateTransportVersionDefinition') { - getMainReleaseBranch().unset() - } - """ - execute('git commit -a -m "setup_main"') - execute('git checkout -b new_branch') referencedTransportVersion("new_tv") when: - def result = runGenerateAndValidateTask("--branches=main").build() - - then: - assertGenerateAndValidateSuccess(result) - assertReferableDefinition("new_tv", "9001000") - assertUpperBound(releaseBranch, "new_tv,9001000") - } - - def "branch from pr targeting main"() { - given: - referencedTransportVersion("new_tv") - - when: - def result = runGenerateAndValidateWithLabels("v9.2.0").build() + def result = runGenerateAndValidateTask().build() then: assertGenerateAndValidateSuccess(result) @@ -424,43 +391,23 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertUpperBound("9.2", "new_tv,8124000") } - def "branch from pr for backport"() { + def "deleted upper bounds files are restored"() { given: - referencedTransportVersion("new_tv") + file("myserver/src/main/resources/transport/upper_bounds/9.2.csv").delete() when: - def result = runGenerateAndValidateWithLabels("v9.2.0", "v9.1.2").build() - - then: - assertGenerateAndValidateSuccess(result) - assertReferableDefinition("new_tv", "8124000,8012002") - assertUpperBound("9.2", "new_tv,8124000") - assertUpperBound("9.1", "new_tv,8012002") - } - - def "branch from pr with other labels"() { - given: - referencedTransportVersion("new_tv") - - when: - def result = runGenerateAndValidateWithLabels(">test", "v9.2.0", ":Core/Infra/Core").build() + def result = runGenerateAndValidateTask().build() then: assertGenerateAndValidateSuccess(result) - assertReferableDefinition("new_tv", "8124000") - assertUpperBound("9.2", "new_tv,8124000") + assertUpperBound("9.2", "existing_92,8123000") } - def "branch from pr default to main"() { - given: - referencedTransportVersion("new_tv") - + def "upper bounds files must exist for backport branches"() { when: - def result = runGenerateAndValidateWithLabels(">test", ":Core/Infra/Core").build() + def result = runGenerateTask("--backport-branches=9.1,8.13,7.17,6.0").buildAndFail() then: - assertGenerateAndValidateSuccess(result) - assertReferableDefinition("new_tv", "8124000") - assertUpperBound("9.2", "new_tv,8124000") + assertGenerateFailure(result, "Missing upper bounds files for branches [6.0, 7.17, 8.13], known branches are [9.0, 9.1, 9.2]") } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index f20f5698a7d4e..b77d040b563d9 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -20,18 +20,13 @@ import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskAction; import org.gradle.api.tasks.options.Option; -import org.gradle.process.ExecOperations; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; - -import javax.inject.Inject; /** * This task generates transport version definition files. These files @@ -55,18 +50,17 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask @Optional @Option( option = "name", - description = "The name of the Transport Version reference for which to generate a definition, e.g. --name=my_new_tv" + description = "The name of the Transport Version definition, e.g. --name=my_new_tv" ) public abstract Property getTransportVersionName(); @Optional @Input @Option( - option = "branches", - description = "The release branches for which to generate IDs, e.g. --branches=main,9.1. " - + "Must always include the main release branch, and backport branches must be contiguous from main." + option = "backport-branches", + description = "The branches this definition will be backported to, e.g. --backport-branches=9.1,8.19" ) - public abstract Property getBranches(); + public abstract Property getBackportBranches(); @Input @Optional @@ -80,13 +74,6 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask @Input public abstract Property getMainReleaseBranch(); - private final ExecOperations execOperations; - - @Inject - public GenerateTransportVersionDefinitionTask(ExecOperations execOperations) { - this.execOperations = execOperations; - } - @TaskAction public void run() throws IOException { TransportVersionResourcesService resources = getResourceService().get(); @@ -96,10 +83,14 @@ public void run() throws IOException { ? getTransportVersionName().get() : findAddedTransportVersionName(resources, referencedNames, changedDefinitionNames); + List mainUpperBounds = resources.getUpperBoundsFromMain(); + Set releaseBranches = getTargetReleaseBranches(); + checkReleaseBranches(mainUpperBounds, releaseBranches); + if (name.isEmpty()) { resetAllLatestFiles(resources); } else { - List ids = updateLatestFiles(resources, name); + List ids = updateUpperBoundsFiles(resources, mainUpperBounds, releaseBranches, name); // (Re)write the definition file. resources.writeReferableDefinition(new TransportVersionDefinition(name, ids)); } @@ -107,45 +98,69 @@ public void run() throws IOException { removeUnusedNamedDefinitions(resources, referencedNames, changedDefinitionNames); } - private List updateLatestFiles(TransportVersionResourcesService resources, String name) throws IOException { - Set targetReleaseBranches = getTargetReleaseBranches(); + private void checkReleaseBranches(List mainUpperBounds, Set releaseBranches) { + Set missingBranches = new HashSet<>(releaseBranches); + List knownReleaseBranches = new ArrayList<>(); + for (TransportVersionUpperBound upperBound : mainUpperBounds) { + knownReleaseBranches.add(upperBound.releaseBranch()); + missingBranches.remove(upperBound.releaseBranch()); + } + if (missingBranches.isEmpty() == false) { + List sortedMissing = missingBranches.stream().sorted().toList(); + List sortedKnown = knownReleaseBranches.stream().sorted().toList(); + throw new IllegalArgumentException("Missing upper bounds files for branches " + sortedMissing + + ", known branches are " + sortedKnown); + } + } + + private List updateUpperBoundsFiles(TransportVersionResourcesService resources, List existingUpperBounds, Set targetReleaseBranches, String name) throws IOException { String mainReleaseBranch = getMainReleaseBranch().get(); int primaryIncrement = getPrimaryIncrement().get(); if (primaryIncrement <= 0) { - throw new IllegalArgumentException("Invalid increment: must be a positive integer > 0"); + throw new IllegalArgumentException("Invalid increment [" + primaryIncrement + "], must be a positive integer"); } List ids = new ArrayList<>(); TransportVersionDefinition existingDefinition = resources.getReferableDefinitionFromMain(name); + for (TransportVersionUpperBound existingUpperBound : existingUpperBounds) { - for (TransportVersionUpperBound upperBound : resources.getUpperBoundsFromMain()) { - if (name.equals(upperBound.name())) { - if (targetReleaseBranches.contains(upperBound.releaseBranch()) == false) { + TransportVersionUpperBound upperBoundToWrite = existingUpperBound; + + // LOGIC + /* + - name may or may not already be set in the existing upper bound + - target release branches may or may not target the upper bound file + */ + + resources.writeUpperBound(upperBoundToWrite); + + if (name.equals(existingUpperBound.name())) { + if (targetReleaseBranches.contains(existingUpperBound.releaseBranch()) == false) { // Here, we aren't targeting this latest file, but need to undo prior updates if the list of minor // versions has changed. We must regenerate this latest file to make this operation idempotent. - resources.writeUpperBound(upperBound); + resources.writeUpperBound(existingUpperBound); } else { - ids.add(upperBound.id()); + ids.add(existingUpperBound.id()); } } else { - if (targetReleaseBranches.contains(upperBound.releaseBranch())) { + if (targetReleaseBranches.contains(existingUpperBound.releaseBranch())) { TransportVersionId targetId = null; if (existingDefinition != null) { for (TransportVersionId id : existingDefinition.ids()) { - if (id.base() == upperBound.id().base()) { + if (id.base() == existingUpperBound.id().base()) { targetId = id; break; } } } if (targetId == null) { - int increment = upperBound.releaseBranch().equals(mainReleaseBranch) ? primaryIncrement : 1; - TransportVersionId id = TransportVersionId.fromInt(upperBound.id().complete() + increment); + int increment = existingUpperBound.releaseBranch().equals(mainReleaseBranch) ? primaryIncrement : 1; + TransportVersionId id = TransportVersionId.fromInt(existingUpperBound.id().complete() + increment); ids.add(id); - resources.writeUpperBound(new TransportVersionUpperBound(upperBound.releaseBranch(), name, id)); + resources.writeUpperBound(new TransportVersionUpperBound(existingUpperBound.releaseBranch(), name, id)); } } else { - resources.writeUpperBound(upperBound); + resources.writeUpperBound(existingUpperBound); } } } @@ -155,33 +170,13 @@ private List updateLatestFiles(TransportVersionResourcesServ } private Set getTargetReleaseBranches() { - if (getBranches().isPresent()) { - return Arrays.stream(getBranches().get().split(",")) - .map(branch -> branch.equals("main") ? getMainReleaseBranch().get() : branch) - .collect(Collectors.toSet()); - } else { - // Look for env var indicating github PR link from CI. - // Use github api to find current labels, filter down to version labels. - // Map version labels to branches. - String prLabels = System.getenv("PULL_REQUEST_LABELS"); - if (prLabels == null) { - throw new RuntimeException("When running outside CI, --branches must be specified"); - } - - Set targetReleaseBranches = new HashSet<>(); - for (String label : prLabels.split(System.lineSeparator())) { - if (label.startsWith("v") == false) { - continue; - } - int firstDot = label.indexOf('.'); - targetReleaseBranches.add(label.substring(1, label.indexOf('.', firstDot + 1))); - } - // if we didn't find any version labels we must be on serverless, so just use the main release releaseBranch - if (targetReleaseBranches.isEmpty()) { - targetReleaseBranches.add(getMainReleaseBranch().get()); - } - return targetReleaseBranches; + Set releaseBranches = new HashSet<>(); + releaseBranches.add(getMainReleaseBranch().get()); + if (getBackportBranches().isPresent()) { + releaseBranches.addAll(List.of(getBackportBranches().get().split(","))); } + + return releaseBranches; } private void resetAllLatestFiles(TransportVersionResourcesService resources) throws IOException { diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy index a40921b19dea4..aa9b9d07c4461 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy @@ -202,13 +202,11 @@ abstract class AbstractGradleFuncTest extends Specification { void execute(String command, File workingDir = testProjectDir.root) { def proc = command.execute(Collections.emptyList(), workingDir) - System.out.println(command) proc.waitFor() if (proc.exitValue()) { System.err.println("Error running command ${command}:") System.err.println("Syserr: " + proc.errorStream.text) } - System.out.println("Output: " + proc.inputStream.text) } File dir(String path) { From e5b8d15f9ec44f10f218758e500f942587a44bb1 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 29 Aug 2025 17:28:06 +0000 Subject: [PATCH 53/68] [CI] Auto commit changes from spotless --- .../GenerateTransportVersionDefinitionTask.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index b77d040b563d9..fb51d9d5979f6 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -48,10 +48,7 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask @Input @Optional - @Option( - option = "name", - description = "The name of the Transport Version definition, e.g. --name=my_new_tv" - ) + @Option(option = "name", description = "The name of the Transport Version definition, e.g. --name=my_new_tv") public abstract Property getTransportVersionName(); @Optional @@ -108,12 +105,18 @@ private void checkReleaseBranches(List mainUpperBoun if (missingBranches.isEmpty() == false) { List sortedMissing = missingBranches.stream().sorted().toList(); List sortedKnown = knownReleaseBranches.stream().sorted().toList(); - throw new IllegalArgumentException("Missing upper bounds files for branches " + sortedMissing + - ", known branches are " + sortedKnown); + throw new IllegalArgumentException( + "Missing upper bounds files for branches " + sortedMissing + ", known branches are " + sortedKnown + ); } } - private List updateUpperBoundsFiles(TransportVersionResourcesService resources, List existingUpperBounds, Set targetReleaseBranches, String name) throws IOException { + private List updateUpperBoundsFiles( + TransportVersionResourcesService resources, + List existingUpperBounds, + Set targetReleaseBranches, + String name + ) throws IOException { String mainReleaseBranch = getMainReleaseBranch().get(); int primaryIncrement = getPrimaryIncrement().get(); if (primaryIncrement <= 0) { From ca78c83056c9a785ce87ecd5300d74defe701cea Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 29 Aug 2025 11:35:53 -0700 Subject: [PATCH 54/68] tests complete --- .../TransportVersionGenerationFuncTest.groovy | 36 +++++----- ...enerateTransportVersionDefinitionTask.java | 71 ++++++++++--------- 2 files changed, 56 insertions(+), 51 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index 97c1331899494..099480ae2d673 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -9,14 +9,10 @@ package org.elasticsearch.gradle.internal.transport -import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.VersionProperties -import org.elasticsearch.gradle.fixtures.WiremockFixture + import org.gradle.testkit.runner.BuildResult import org.gradle.testkit.runner.TaskOutcome -import java.nio.charset.StandardCharsets - class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTest { def runGenerateAndValidateTask(String... additionalArgs) { @@ -94,7 +90,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertUpperBound("9.2", "new_tv,8124000") } - def "invalid changes to a latest file should be reverted"() { + def "invalid changes to a upper bounds should be reverted"() { given: transportVersionUpperBound("9.2", "modification", "9000000") @@ -106,7 +102,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertUpperBound("9.2", "existing_92,8123000") } - def "invalid changes to multiple latest files should be reverted"() { + def "invalid changes to multiple upper bounds should be reverted"() { given: transportVersionUpperBound("9.2", "modification", "9000000") transportVersionUpperBound("9.1", "modification", "9000000") @@ -198,8 +194,9 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertUpperBound("9.2", "test_tv,8124000") } - def "a latest file can be regenerated"() { + def "an upper bound can be regenerated"() { given: + file("myserver/src/main/resources/transport/upper_bounds/9.2.csv").delete() referableAndReferencedTransportVersion("test_tv", "8124000") when: @@ -310,7 +307,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertUpperBound("9.2", "test_tv,8124000") } - def "can add backport to an existing definition"() { + def "can add backport to latest upper bound"() { when: def result = runGenerateAndValidateTask("--name=existing_92", "--backport-branches=9.1,9.0").build() @@ -322,17 +319,24 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes assertUpperBound("9.0", "existing_92,8000001") } - def "increment must be positive"() { + def "can add backport to older definition"() { given: - referencedTransportVersion("new_tv") + execute("git checkout main") + referableAndReferencedTransportVersion("latest_tv", "8124000,8012002") + transportVersionUpperBound("9.2", "latest_tv", "8124000") + transportVersionUpperBound("9.1", "latest_tv", "8012002") + execute("git commit -a -m added") + execute("git checkout -b mybranch") when: - def result = runGenerateTask("--increment=0").buildAndFail() + def result = runGenerateAndValidateTask("--name=existing_92", "--backport-branches=9.1,9.0").build() then: - assertGenerateFailure(result, "Invalid increment [0], must be a positive integer") - assertReferableDefinitionDoesNotExist("new_tv") - assertUpperBound("9.2", "existing_92,8123000") + assertGenerateAndValidateSuccess(result) + assertReferableDefinition("existing_92", "8123000,8012001,8000001") + assertUpperBound("9.2", "latest_tv,8124000") + assertUpperBound("9.1", "latest_tv,8012002") + assertUpperBound("9.0", "existing_92,8000001") } def "a different increment can be specified"() { @@ -361,7 +365,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes def result = runGenerateTask("--increment=0").buildAndFail() then: - assertOutputContains(result.output, "Invalid increment: must be a positive integer > 0") + assertOutputContains(result.output, "Invalid increment 0, must be a positive integer") } def "a new definition exists and is in the latest file, but the version id is wrong and needs to be updated"(){ diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index fb51d9d5979f6..e69af607cb702 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -120,51 +120,36 @@ private List updateUpperBoundsFiles( String mainReleaseBranch = getMainReleaseBranch().get(); int primaryIncrement = getPrimaryIncrement().get(); if (primaryIncrement <= 0) { - throw new IllegalArgumentException("Invalid increment [" + primaryIncrement + "], must be a positive integer"); + throw new IllegalArgumentException("Invalid increment " + primaryIncrement + ", must be a positive integer"); } List ids = new ArrayList<>(); TransportVersionDefinition existingDefinition = resources.getReferableDefinitionFromMain(name); for (TransportVersionUpperBound existingUpperBound : existingUpperBounds) { - TransportVersionUpperBound upperBoundToWrite = existingUpperBound; + String releaseBranch = existingUpperBound.releaseBranch(); + boolean targetThisBranch = targetReleaseBranches.contains(existingUpperBound.releaseBranch()); - // LOGIC - /* - - name may or may not already be set in the existing upper bound - - target release branches may or may not target the upper bound file - */ - - resources.writeUpperBound(upperBoundToWrite); + // Case: Targeting this upper bound file and it already points at the name we're generating + if (targetThisBranch && name.equals(existingUpperBound.name())) { + ids.add(existingUpperBound.id()); + continue; + } - if (name.equals(existingUpperBound.name())) { - if (targetReleaseBranches.contains(existingUpperBound.releaseBranch()) == false) { - // Here, we aren't targeting this latest file, but need to undo prior updates if the list of minor - // versions has changed. We must regenerate this latest file to make this operation idempotent. - resources.writeUpperBound(existingUpperBound); - } else { - ids.add(existingUpperBound.id()); + if (targetThisBranch) { + // Case: targeting this branch, find an existing id for this branch if it exists + TransportVersionId targetId = maybeGetExistingId(existingUpperBound, existingDefinition); + if (targetId == null) { + // Case: an id doesn't yet exist for this branch, so create one + int increment = releaseBranch.equals(mainReleaseBranch) ? primaryIncrement : 1; + targetId = TransportVersionId.fromInt(existingUpperBound.id().complete() + increment); + var newUpperBound = new TransportVersionUpperBound(releaseBranch, name, targetId); + resources.writeUpperBound(newUpperBound); } + ids.add(targetId); } else { - if (targetReleaseBranches.contains(existingUpperBound.releaseBranch())) { - TransportVersionId targetId = null; - if (existingDefinition != null) { - for (TransportVersionId id : existingDefinition.ids()) { - if (id.base() == existingUpperBound.id().base()) { - targetId = id; - break; - } - } - } - if (targetId == null) { - int increment = existingUpperBound.releaseBranch().equals(mainReleaseBranch) ? primaryIncrement : 1; - TransportVersionId id = TransportVersionId.fromInt(existingUpperBound.id().complete() + increment); - ids.add(id); - resources.writeUpperBound(new TransportVersionUpperBound(existingUpperBound.releaseBranch(), name, id)); - } - } else { - resources.writeUpperBound(existingUpperBound); - } + // Default case: we're not targeting this branch so reset it + resources.writeUpperBound(existingUpperBound); } } @@ -230,4 +215,20 @@ private String findAddedTransportVersionName( } } } + + private TransportVersionId maybeGetExistingId(TransportVersionUpperBound upperBound, TransportVersionDefinition existingDefinition) { + if (existingDefinition == null) { + return null; + } + if (upperBound.releaseBranch().equals(getMainReleaseBranch().get())) { + return existingDefinition.ids().get(0); // main is always the primary id + } + for (TransportVersionId id : existingDefinition.ids()) { + if (id.base() == upperBound.id().base()) { + return id; + } + } + return null; // no id for this release branch + } + } From 8798cfcd79f7e06e918dbc32ba1982ad33397acf Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 2 Sep 2025 17:23:31 -0700 Subject: [PATCH 55/68] feedback --- .../TransportVersionGenerationFuncTest.groovy | 3 +- .../TransportVersionValidationFuncTest.groovy | 2 +- ...enerateTransportVersionDefinitionTask.java | 33 ++++++------ .../transport/TransportVersionId.java | 50 ++----------------- .../TransportVersionResourcesPlugin.java | 1 + ...ValidateTransportVersionResourcesTask.java | 5 +- 6 files changed, 28 insertions(+), 66 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index 099480ae2d673..46eecc75ffa40 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -345,11 +345,12 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes file("myserver/build.gradle") << """ tasks.named('validateTransportVersionResources') { shouldValidateDensity = false + shouldValidatePrimaryIdNotPatch = false } """ when: - def result = runGenerateAndValidateTask("--backport-branches=9.2", "--increment=100").build() + def result = runGenerateAndValidateTask("--increment=100").build() then: assertGenerateAndValidateSuccess(result) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy index d77ccf79ac887..36be4c4d94bff 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy @@ -26,7 +26,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes def "definitions must be referenced"() { given: javaSource("myplugin", "org.elasticsearch.plugin", "MyPlugin", - "import org.elasticsearch.TransportVersion;", """ + "import org.elasticsearch.TransportVersion;", """ static final TransportVersion dne = TransportVersion.fromName("dne"); """) when: diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index e69af607cb702..c17989b1fe7a5 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -51,8 +51,8 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask @Option(option = "name", description = "The name of the Transport Version definition, e.g. --name=my_new_tv") public abstract Property getTransportVersionName(); - @Optional @Input + @Optional @Option( option = "backport-branches", description = "The branches this definition will be backported to, e.g. --backport-branches=9.1,8.19" @@ -85,9 +85,9 @@ public void run() throws IOException { checkReleaseBranches(mainUpperBounds, releaseBranches); if (name.isEmpty()) { - resetAllLatestFiles(resources); + resetAllUpperBounds(resources); } else { - List ids = updateUpperBoundsFiles(resources, mainUpperBounds, releaseBranches, name); + List ids = updateUpperBounds(resources, mainUpperBounds, releaseBranches, name); // (Re)write the definition file. resources.writeReferableDefinition(new TransportVersionDefinition(name, ids)); } @@ -111,7 +111,7 @@ private void checkReleaseBranches(List mainUpperBoun } } - private List updateUpperBoundsFiles( + private List updateUpperBounds( TransportVersionResourcesService resources, List existingUpperBounds, Set targetReleaseBranches, @@ -126,19 +126,11 @@ private List updateUpperBoundsFiles( TransportVersionDefinition existingDefinition = resources.getReferableDefinitionFromMain(name); for (TransportVersionUpperBound existingUpperBound : existingUpperBounds) { - String releaseBranch = existingUpperBound.releaseBranch(); - boolean targetThisBranch = targetReleaseBranches.contains(existingUpperBound.releaseBranch()); - // Case: Targeting this upper bound file and it already points at the name we're generating - if (targetThisBranch && name.equals(existingUpperBound.name())) { - ids.add(existingUpperBound.id()); - continue; - } - - if (targetThisBranch) { + if (targetReleaseBranches.contains(releaseBranch)) { // Case: targeting this branch, find an existing id for this branch if it exists - TransportVersionId targetId = maybeGetExistingId(existingUpperBound, existingDefinition); + TransportVersionId targetId = maybeGetExistingId(existingUpperBound, existingDefinition, name); if (targetId == null) { // Case: an id doesn't yet exist for this branch, so create one int increment = releaseBranch.equals(mainReleaseBranch) ? primaryIncrement : 1; @@ -167,7 +159,7 @@ private Set getTargetReleaseBranches() { return releaseBranches; } - private void resetAllLatestFiles(TransportVersionResourcesService resources) throws IOException { + private void resetAllUpperBounds(TransportVersionResourcesService resources) throws IOException { // TODO: this should also _delete_ extraneous files from latest? for (TransportVersionUpperBound upperBound : resources.getUpperBoundsFromMain()) { resources.writeUpperBound(upperBound); @@ -216,13 +208,20 @@ private String findAddedTransportVersionName( } } - private TransportVersionId maybeGetExistingId(TransportVersionUpperBound upperBound, TransportVersionDefinition existingDefinition) { + private TransportVersionId maybeGetExistingId(TransportVersionUpperBound upperBound, TransportVersionDefinition existingDefinition, String name) { if (existingDefinition == null) { + // the name doesn't yet exist, so there is no id to return return null; } + if (upperBound.name().equals(name)) { + // the name exists and this upper bound already points at it + return upperBound.id(); + } if (upperBound.releaseBranch().equals(getMainReleaseBranch().get())) { - return existingDefinition.ids().get(0); // main is always the primary id + // the upper bound is for main, so return the primary id + return existingDefinition.ids().get(0); } + // the upper bound is for a non-main branch, so find the id with the same base for (TransportVersionId id : existingDefinition.ids()) { if (id.base() == upperBound.id().base()) { return id; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionId.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionId.java index c0e2ea0e85f8c..7b90907b9c23f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionId.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionId.java @@ -9,57 +9,19 @@ package org.elasticsearch.gradle.internal.transport; -record TransportVersionId(int complete, int major, int server, int subsidiary, int patch) implements Comparable { - - /** - * The TV format: - *

- * MM_NNN_S_PP - *

- * M - The major version of Elasticsearch - * NNN - The server version part - * S - The subsidiary version part. It should always be 0 here, it is only used in subsidiary repositories. - * PP - The patch version part - */ - public enum Component { - MAJOR(1_000_0_00, 2), - SERVER(1_0_00, 3), - SUBSIDIARY(1_00, 1), - PATCH(1, 2); - - private final int value; - private final int max; - - Component(int value, int numDigits) { - this.value = value; - this.max = (int) Math.pow(10, numDigits); - } - } +record TransportVersionId(int complete, int base, int patch) implements Comparable { public static TransportVersionId fromInt(int complete) { - int patch = complete % Component.PATCH.max; - int subsidiary = (complete / Component.SUBSIDIARY.value) % Component.SUBSIDIARY.max; - int server = (complete / Component.SERVER.value) % Component.SERVER.max; - int major = complete / Component.MAJOR.value; + int patch = complete % 1000; + int base = complete - patch; - return new TransportVersionId(complete, major, server, subsidiary, patch); + return new TransportVersionId(complete, base, patch); } static TransportVersionId fromString(String s) { return fromInt(Integer.parseInt(s)); } - public TransportVersionId bumpComponent(Component component) { - int zeroesCleared = (complete / component.value) * component.value; - int newId = zeroesCleared + component.value; - if ((newId / component.value) % component.max == 0) { - throw new IllegalStateException( - "Insufficient" + component.name() + " version section in TransportVersion: " + complete + ", Cannot bump." - ); - } - return fromInt(newId); - } - @Override public int compareTo(TransportVersionId o) { // note: this is descending order so the arguments are reversed @@ -70,8 +32,4 @@ public int compareTo(TransportVersionId o) { public String toString() { return Integer.toString(complete); } - - public int base() { - return (complete / 1000) * 1000; - } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java index 9c311f299e053..7c80d2c52804e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java @@ -57,6 +57,7 @@ public void apply(Project project) { t.setDescription("Validates that all transport version resources are internally consistent with each other"); t.getReferencesFiles().setFrom(tvReferencesConfig); t.getShouldValidateDensity().convention(true); + t.getShouldValidatePrimaryIdNotPatch().convention(true); }); project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(validateTask)); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java index 0804e8910faf0..97820aab29d82 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java @@ -57,6 +57,9 @@ public Path getResourcesDir() { @Input public abstract Property getShouldValidateDensity(); + @Input + public abstract Property getShouldValidatePrimaryIdNotPatch(); + private record IdAndDefinition(TransportVersionId id, TransportVersionDefinition definition) {} private static final Pattern NAME_FORMAT = Pattern.compile("[a-z0-9_]+"); @@ -153,7 +156,7 @@ private void validateNamedDefinition(TransportVersionDefinition definition, Set< TransportVersionId id = definition.ids().get(ndx); if (ndx == 0) { - if (id.patch() != 0) { + if (getShouldValidatePrimaryIdNotPatch().get() && id.patch() != 0) { throwDefinitionFailure(definition, "has patch version " + id.complete() + " as primary id"); } } else { From aa323b62952099a8d0674a33d9519e8d508cd120 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 2 Sep 2025 17:25:22 -0700 Subject: [PATCH 56/68] spotless --- .../transport/GenerateTransportVersionDefinitionTask.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index c17989b1fe7a5..1a7bcd6d344a7 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -208,7 +208,11 @@ private String findAddedTransportVersionName( } } - private TransportVersionId maybeGetExistingId(TransportVersionUpperBound upperBound, TransportVersionDefinition existingDefinition, String name) { + private TransportVersionId maybeGetExistingId( + TransportVersionUpperBound upperBound, + TransportVersionDefinition existingDefinition, + String name + ) { if (existingDefinition == null) { // the name doesn't yet exist, so there is no id to return return null; From a5eb9f8833040e4a156a0bcfba61100d603bda8c Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 2 Sep 2025 17:32:41 -0700 Subject: [PATCH 57/68] cleanup --- ...enerateTransportVersionDefinitionTask.java | 63 ++++++++++--------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index 1a7bcd6d344a7..aea8a1ad49db8 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -76,9 +76,7 @@ public void run() throws IOException { TransportVersionResourcesService resources = getResourceService().get(); Set referencedNames = TransportVersionReference.collectNames(getReferencesFiles()); List changedDefinitionNames = resources.getChangedReferableDefinitionNames(); - String name = getTransportVersionName().isPresent() - ? getTransportVersionName().get() - : findAddedTransportVersionName(resources, referencedNames, changedDefinitionNames); + String name = getTargetName(resources, referencedNames, changedDefinitionNames); List mainUpperBounds = resources.getUpperBoundsFromMain(); Set releaseBranches = getTargetReleaseBranches(); @@ -149,6 +147,37 @@ private List updateUpperBounds( return ids; } + private String getTargetName(TransportVersionResourcesService resources, + Set referencedNames, + List changedDefinitions) { + if (getTransportVersionName().isPresent()) { + // an explicit name was passed in, so use it + return getTransportVersionName().get(); + } + + // First check for unreferenced names. We only care about the first one. If there is more than one + // validation will fail later and the developer will have to remove one. When that happens, generation + // will re-run and we will fixup the state to use whatever new name remains. + for (String referencedName : referencedNames) { + if (resources.referableDefinitionExists(referencedName) == false) { + return referencedName; + } + } + + // Since we didn't find any missing names, we use the first changed name. If there is more than + // one changed name, validation will fail later, just as above. + if (changedDefinitions.isEmpty()) { + return ""; + } else { + String changedDefinitionName = changedDefinitions.getFirst(); + if (referencedNames.contains(changedDefinitionName)) { + return changedDefinitionName; + } else { + return ""; // the changed name is unreferenced, so go into "reset mode" + } + } + } + private Set getTargetReleaseBranches() { Set releaseBranches = new HashSet<>(); releaseBranches.add(getMainReleaseBranch().get()); @@ -180,34 +209,6 @@ private void removeUnusedNamedDefinitions( } } - private String findAddedTransportVersionName( - TransportVersionResourcesService resources, - Set referencedNames, - List changedDefinitions - ) { - // First check for unreferenced names. We only care about the first one. If there is more than one - // validation will fail later and the developer will have to remove one. When that happens, generation - // will re-run and we will fixup the state to use whatever new name remains. - for (String referencedName : referencedNames) { - if (resources.referableDefinitionExists(referencedName) == false) { - return referencedName; - } - } - - // Since we didn't find any missing names, we use the first changed name. If there is more than - // one changed name, validation will fail later, just as above. - if (changedDefinitions.isEmpty()) { - return ""; - } else { - String changedDefinitionName = changedDefinitions.getFirst(); - if (referencedNames.contains(changedDefinitionName)) { - return changedDefinitionName; - } else { - return ""; // the changed name is unreferenced, so go into "reset mode" - } - } - } - private TransportVersionId maybeGetExistingId( TransportVersionUpperBound upperBound, TransportVersionDefinition existingDefinition, From 3d84dde501723e44225495d0335ffa67d4e0dd9f Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 3 Sep 2025 00:39:36 +0000 Subject: [PATCH 58/68] [CI] Auto commit changes from spotless --- .../transport/GenerateTransportVersionDefinitionTask.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index aea8a1ad49db8..258ec77a8441d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -147,9 +147,7 @@ private List updateUpperBounds( return ids; } - private String getTargetName(TransportVersionResourcesService resources, - Set referencedNames, - List changedDefinitions) { + private String getTargetName(TransportVersionResourcesService resources, Set referencedNames, List changedDefinitions) { if (getTransportVersionName().isPresent()) { // an explicit name was passed in, so use it return getTransportVersionName().get(); From 853498f017974e1233139951fc8a20929744730b Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 4 Sep 2025 07:40:19 -0700 Subject: [PATCH 59/68] more feedback --- .../transport/TransportVersionResourcesPlugin.java | 8 +++++--- .../transport/TransportVersionResourcesService.java | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java index 7c80d2c52804e..814722f852cb6 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java @@ -31,6 +31,8 @@ public void apply(Project project) { var psService = project.getPlugins().apply(ProjectSubscribeServicePlugin.class).getService(); var resourceRoot = getResourceRoot(project); + String taskGroup = "Transport Versions"; + project.getGradle() .getSharedServices() .registerIfAbsent("transportVersionResources", TransportVersionResourcesService.class, spec -> { @@ -53,7 +55,7 @@ public void apply(Project project) { var validateTask = project.getTasks() .register("validateTransportVersionResources", ValidateTransportVersionResourcesTask.class, t -> { - t.setGroup("Transport Versions"); + t.setGroup(taskGroup); t.setDescription("Validates that all transport version resources are internally consistent with each other"); t.getReferencesFiles().setFrom(tvReferencesConfig); t.getShouldValidateDensity().convention(true); @@ -63,7 +65,7 @@ public void apply(Project project) { var generateManifestTask = project.getTasks() .register("generateTransportVersionManifest", GenerateTransportVersionManifestTask.class, t -> { - t.setGroup("Transport Versions"); + t.setGroup(taskGroup); t.setDescription("Generate a manifest resource for all transport version definitions"); t.getManifestFile().set(project.getLayout().getBuildDirectory().file("generated-resources/manifest.txt")); }); @@ -73,7 +75,7 @@ public void apply(Project project) { var generateDefinitionsTask = project.getTasks() .register("generateTransportVersionDefinition", GenerateTransportVersionDefinitionTask.class, t -> { - t.setGroup("Transport Versions"); + t.setGroup(taskGroup); t.setDescription("(Re)generates a transport version definition file"); t.getReferencesFiles().setFrom(tvReferencesConfig); t.getPrimaryIncrement().convention(1000); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java index 6ba3b399383b3..98c8f5dc0ebc3 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java @@ -112,6 +112,7 @@ TransportVersionDefinition getReferableDefinitionFromMain(String name) { return getMainFile(resourcePath, TransportVersionDefinition::fromString); } + /** Get the definition names which have local changes relative to the main branch */ List getChangedReferableDefinitionNames() { List changedDefinitions = new ArrayList<>(); String referablePrefix = REFERABLE_DIR.toString(); From b3d3a7509123b4d775ccd428713551f095815311 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 4 Sep 2025 08:57:21 -0700 Subject: [PATCH 60/68] add tests for detecting name from staged/committed changes --- .../TransportVersionGenerationFuncTest.groovy | 30 ++++++++++++++++++- ...enerateTransportVersionDefinitionTask.java | 1 + .../TransportVersionResourcesService.java | 3 +- .../fixtures/AbstractGradleFuncTest.groovy | 8 +++-- 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index 46eecc75ffa40..f3f82130a68ac 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -297,7 +297,6 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes transportVersionUpperBound("9.2", "test_tv", "8124000") referableAndReferencedTransportVersion("test_tv", "8124000") - when: def result = runGenerateAndValidateTask().build() @@ -415,4 +414,33 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes then: assertGenerateFailure(result, "Missing upper bounds files for branches [6.0, 7.17, 8.13], known branches are [9.0, 9.1, 9.2]") } + + def "name can be found from committed definition"() { + given: + referableAndReferencedTransportVersion("new_tv", "8123000") + execute("git add .") + execute("git commit -m added") + + when: + def result = runGenerateAndValidateTask().build() + + then: + assertGenerateAndValidateSuccess(result) + assertUpperBound("9.2", "new_tv,8124000") + assertReferableDefinition("new_tv", "8124000") + } + + def "name can be found from staged definition"() { + given: + referableAndReferencedTransportVersion("new_tv", "8123000") + execute("git add .") + + when: + def result = runGenerateAndValidateTask().build() + + then: + assertGenerateAndValidateSuccess(result) + assertUpperBound("9.2", "new_tv,8124000") + assertReferableDefinition("new_tv", "8124000") + } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index 258ec77a8441d..ae2dfaa0c9cc2 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -82,6 +82,7 @@ public void run() throws IOException { Set releaseBranches = getTargetReleaseBranches(); checkReleaseBranches(mainUpperBounds, releaseBranches); + getLogger().lifecycle("Generating transport version name: " + name); if (name.isEmpty()) { resetAllUpperBounds(resources); } else { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java index 98c8f5dc0ebc3..01aec6e677f42 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java @@ -120,7 +120,8 @@ List getChangedReferableDefinitionNames() { if (changedPath.contains(referablePrefix) == false) { continue; } - String name = changedPath.substring(referablePrefix.length() + 1 /* skip slash */, changedPath.length() - 4 /* .csv */); + int lastSlashNdx = changedPath.lastIndexOf('/'); + String name = changedPath.substring(lastSlashNdx + 1, changedPath.length() - 4 /* .csv */); changedDefinitions.add(name); } return changedDefinitions; diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy index 447f0b8af496f..df14753375c64 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy @@ -20,6 +20,7 @@ import org.elasticsearch.gradle.internal.test.NormalizeOutputGradleRunner import org.elasticsearch.gradle.internal.test.TestResultExtension import org.gradle.testkit.runner.BuildResult import org.gradle.testkit.runner.GradleRunner +import org.gradle.tooling.BuildException import org.junit.Rule import org.junit.rules.TemporaryFolder @@ -214,8 +215,11 @@ abstract class AbstractGradleFuncTest extends Specification { def proc = command.execute(Collections.emptyList(), workingDir) proc.waitFor() if (proc.exitValue()) { - System.err.println("Error running command ${command}:") - System.err.println("Syserr: " + proc.errorStream.text) + String msg = """Error running command ${command}: + Sysout: ${proc.inputStream.text} + Syserr: ${proc.errorStream.text} + """ + throw new RuntimeException(msg) } } From 0c642b941ec76bf8ae06131328ba90a82db12b4a Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 4 Sep 2025 14:05:16 -0700 Subject: [PATCH 61/68] rename releaseBranch to name --- .../AbstractTransportVersionFuncTest.groovy | 10 +- .../TransportVersionGenerationFuncTest.groovy | 2 +- .../TransportVersionValidationFuncTest.groovy | 30 ++--- ...enerateTransportVersionDefinitionTask.java | 113 +++++++++--------- .../transport/TransportVersionDefinition.java | 2 +- .../TransportVersionResourcesPlugin.java | 2 +- .../TransportVersionResourcesService.java | 24 ++-- .../transport/TransportVersionUpperBound.java | 4 +- ...ValidateTransportVersionResourcesTask.java | 60 +++++----- 9 files changed, 122 insertions(+), 125 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy index 31d007192df0c..a9b7f2c2398b6 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy @@ -93,8 +93,8 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest { assert file("myserver/src/main/resources/transport/definitions/referable/${name}.csv").exists() == false } - void assertUpperBound(String releaseBranch, String content) { - assert file("myserver/src/main/resources/transport/upper_bounds/${releaseBranch}.csv").text.strip() == content + void assertUpperBound(String name, String content) { + assert file("myserver/src/main/resources/transport/upper_bounds/${name}.csv").text.strip() == content } def setup() { @@ -111,7 +111,7 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest { apply plugin: 'elasticsearch.transport-version-resources' tasks.named('generateTransportVersionDefinition') { - mainReleaseBranch = '9.2' + currentUpperBoundName = '9.2' } """ referableTransportVersion("existing_91", "8012000") @@ -122,7 +122,7 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest { transportVersionUpperBound("9.0", "initial_9_0_0", "8000000") // a mock version of TransportVersion, just here so we can compile Dummy.java et al javaSource("myserver", "org.elasticsearch", "TransportVersion", "", """ - public static TransportVersion fromName(String name) { + public static TransportVersion fromName(String definitionName) { return null; } """) @@ -148,7 +148,7 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest { void setupLocalGitRepo() { execute("git init") execute('git config user.email "build-tool@elastic.co"') - execute('git config user.name "Build tool"') + execute('git config user.definitionName "Build tool"') execute("git add .") execute('git commit -m "Initial"') } diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index f3f82130a68ac..49f849a9996d3 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -263,7 +263,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes existing_92,8123000 ======= second_tv,8123000 - >>>>>> releaseBranch + >>>>>> name """.strip() referableAndReferencedTransportVersion("second_tv", "8123000") diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy index 36be4c4d94bff..3772eb1373fd3 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy @@ -53,7 +53,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes def result = validateResourcesFails() then: assertValidateResourcesFailure(result, "Transport version definition file " + - "[myserver/src/main/resources/transport/definitions/referable/${name}.csv] does not have a valid name, " + + "[myserver/src/main/resources/transport/definitions/referable/${name}.csv] does not have a valid definitionName, " + "must be lowercase alphanumeric and underscore") where: @@ -87,7 +87,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes def result = validateResourcesFails() then: assertValidateResourcesFailure(result, "Transport version definition file " + - "[myserver/src/main/resources/transport/definitions/referable/existing_92.csv] contains id 8123000 already defined in " + + "[myserver/src/main/resources/transport/definitions/referable/existing_92.csv] contains definitionId 8123000 already defined in " + "[myserver/src/main/resources/transport/definitions/referable/duplicate.csv]") } @@ -98,7 +98,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes def result = validateResourcesFails() then: assertValidateResourcesFailure(result, "Transport version definition file " + - "[myserver/src/main/resources/transport/definitions/referable/patched.csv] contains bwc id [8100000] with a patch part of 0") + "[myserver/src/main/resources/transport/definitions/referable/patched.csv] contains bwc definitionId [8100000] with a patch part of 0") } def "definitions have primary ids which cannot change"() { @@ -108,7 +108,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes def result = validateResourcesFails() then: assertValidateResourcesFailure(result, "Transport version definition file " + - "[myserver/src/main/resources/transport/definitions/referable/existing_92.csv] has modified primary id from 8123000 to 8500000") + "[myserver/src/main/resources/transport/definitions/referable/existing_92.csv] has modified primary definitionId from 8123000 to 8500000") } def "cannot change committed ids to a branch"() { @@ -118,7 +118,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes def result = validateResourcesFails() then: assertValidateResourcesFailure(result, "Transport version definition file " + - "[myserver/src/main/resources/transport/definitions/referable/existing_92.csv] modifies existing patch id from 8012001 to 8012002") + "[myserver/src/main/resources/transport/definitions/referable/existing_92.csv] modifies existing patch definitionId from 8012001 to 8012002") } def "upper bounds files must reference defined name"() { @@ -128,7 +128,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes def result = validateResourcesFails() then: assertValidateResourcesFailure(result, "Transport version upper bound file " + - "[myserver/src/main/resources/transport/upper_bounds/9.2.csv] contains transport version name [dne] which is not defined") + "[myserver/src/main/resources/transport/upper_bounds/9.2.csv] contains transport version definitionName [dne] which is not defined") } def "upper bound files id must exist in definition"() { @@ -138,7 +138,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes def result = validateResourcesFails() then: assertValidateResourcesFailure(result, "Transport version upper bound file " + - "[myserver/src/main/resources/transport/upper_bounds/9.2.csv] has id 8124000 which is not in definition " + + "[myserver/src/main/resources/transport/upper_bounds/9.2.csv] has definitionId 8124000 which is not in definition " + "[myserver/src/main/resources/transport/definitions/referable/existing_92.csv]") } @@ -152,8 +152,8 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes def result = validateResourcesFails() then: assertValidateResourcesFailure(result, "Transport version upper bound file " + - "[myserver/src/main/resources/transport/upper_bounds/9.0.csv] has id 8110001 from [seemingly_latest] with base 8110000 " + - "but another id 8110002 from [actual_latest] is later for that base") + "[myserver/src/main/resources/transport/upper_bounds/9.0.csv] has definitionId 8110001 from [seemingly_latest] with base 8110000 " + + "but another definitionId 8110002 from [actual_latest] is later for that base") } def "upper bound files cannot change base id"() { @@ -165,7 +165,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes def result = validateResourcesFails() then: assertValidateResourcesFailure(result, "Transport version upper bound file " + - "[myserver/src/main/resources/transport/upper_bounds/9.1.csv] modifies base id from 8012000 to 8013000") + "[myserver/src/main/resources/transport/upper_bounds/9.1.csv] modifies base definitionId from 8012000 to 8013000") } def "ids must be dense"() { @@ -176,7 +176,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes when: def result = validateResourcesFails() then: - assertValidateResourcesFailure(result, "Transport version base id 8013000 is missing patch ids between 8013000 and 8013002") + assertValidateResourcesFailure(result, "Transport version base definitionId 8013000 is missing patch ids between 8013000 and 8013002") } def "primary id must not be patch version"() { @@ -186,7 +186,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes def result = validateResourcesFails() then: assertValidateResourcesFailure(result, "Transport version definition file " + - "[myserver/src/main/resources/transport/definitions/referable/patch.csv] has patch version 8015001 as primary id") + "[myserver/src/main/resources/transport/definitions/referable/patch.csv] has patch version 8015001 as primary definitionId") } def "unreferable directory is optional"() { @@ -217,7 +217,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes then: assertValidateResourcesFailure(result, "Transport version definition file " + "[myserver/src/main/resources/transport/definitions/referable/existing_92.csv] " + - "has same name as unreferable definition " + + "has same definitionName as unreferable definition " + "[myserver/src/main/resources/transport/definitions/unreferable/existing_92.csv]") } @@ -238,7 +238,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes then: assertValidateResourcesFailure(result, "Transport version definition file " + "[myserver/src/main/resources/transport/definitions/referable/some_tv.csv] " + - "has the highest transport version id [10000000] but is not present in any upper bounds files") + "has the highest transport version definitionId [10000000] but is not present in any upper bounds files") } def "highest id in an unreferable definition should exist in an upper bounds file"() { @@ -250,6 +250,6 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes // TODO: this should be _unreferable_ in the error message, but will require some rework assertValidateResourcesFailure(result, "Transport version definition file " + "[myserver/src/main/resources/transport/definitions/referable/initial_10.0.0.csv] " + - "has the highest transport version id [10000000] but is not present in any upper bounds files") + "has the highest transport version definitionId [10000000] but is not present in any upper bounds files") } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index ae2dfaa0c9cc2..fafdebb7bea05 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -32,7 +32,7 @@ * This task generates transport version definition files. These files * are runtime resources that TransportVersion loads statically. * They contain a comma separated list of integer ids. Each file is named the same - * as the transport version name itself (with the .csv suffix). + * as the transport version definitionName itself (with the .csv suffix). */ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask { @@ -48,7 +48,7 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask @Input @Optional - @Option(option = "name", description = "The name of the Transport Version definition, e.g. --name=my_new_tv") + @Option(option = "name", description = "The definitionName of the Transport Version definition, e.g. --definitionName=my_new_tv") public abstract Property getTransportVersionName(); @Input @@ -61,62 +61,45 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask @Input @Optional - @Option(option = "increment", description = "The amount to increment the primary id for the main releaseBranch") + @Option(option = "increment", description = "The amount to increment the primary definitionId for the main name") public abstract Property getPrimaryIncrement(); /** - * The version number currently associated with the `main` branch, (e.g. as returned by - * VersionProperties.getElasticsearchVersion()). + * The name of the upper bounds file which will be used at runtime on the current branch. Normally + * this equates to VersionProperties.getElasticsearchVersion(). */ @Input - public abstract Property getMainReleaseBranch(); + public abstract Property getCurrentUpperBoundName(); @TaskAction public void run() throws IOException { TransportVersionResourcesService resources = getResourceService().get(); Set referencedNames = TransportVersionReference.collectNames(getReferencesFiles()); List changedDefinitionNames = resources.getChangedReferableDefinitionNames(); - String name = getTargetName(resources, referencedNames, changedDefinitionNames); + String targetDefinitionName = getTargetDefinitionName(resources, referencedNames, changedDefinitionNames); List mainUpperBounds = resources.getUpperBoundsFromMain(); - Set releaseBranches = getTargetReleaseBranches(); - checkReleaseBranches(mainUpperBounds, releaseBranches); + Set targetUpperBoundNames = getTargetUpperBoundNames(mainUpperBounds); - getLogger().lifecycle("Generating transport version name: " + name); - if (name.isEmpty()) { + getLogger().lifecycle("Generating transport version name: " + targetDefinitionName); + if (targetDefinitionName.isEmpty()) { resetAllUpperBounds(resources); } else { - List ids = updateUpperBounds(resources, mainUpperBounds, releaseBranches, name); + List ids = updateUpperBounds(resources, mainUpperBounds, targetUpperBoundNames, targetDefinitionName); // (Re)write the definition file. - resources.writeReferableDefinition(new TransportVersionDefinition(name, ids)); + resources.writeReferableDefinition(new TransportVersionDefinition(targetDefinitionName, ids)); } removeUnusedNamedDefinitions(resources, referencedNames, changedDefinitionNames); } - private void checkReleaseBranches(List mainUpperBounds, Set releaseBranches) { - Set missingBranches = new HashSet<>(releaseBranches); - List knownReleaseBranches = new ArrayList<>(); - for (TransportVersionUpperBound upperBound : mainUpperBounds) { - knownReleaseBranches.add(upperBound.releaseBranch()); - missingBranches.remove(upperBound.releaseBranch()); - } - if (missingBranches.isEmpty() == false) { - List sortedMissing = missingBranches.stream().sorted().toList(); - List sortedKnown = knownReleaseBranches.stream().sorted().toList(); - throw new IllegalArgumentException( - "Missing upper bounds files for branches " + sortedMissing + ", known branches are " + sortedKnown - ); - } - } - private List updateUpperBounds( TransportVersionResourcesService resources, List existingUpperBounds, - Set targetReleaseBranches, + Set targetUpperBoundNames, String name ) throws IOException { - String mainReleaseBranch = getMainReleaseBranch().get(); + String currentUpperBoundName = getCurrentUpperBoundName().get(); int primaryIncrement = getPrimaryIncrement().get(); if (primaryIncrement <= 0) { throw new IllegalArgumentException("Invalid increment " + primaryIncrement + ", must be a positive integer"); @@ -125,16 +108,16 @@ private List updateUpperBounds( TransportVersionDefinition existingDefinition = resources.getReferableDefinitionFromMain(name); for (TransportVersionUpperBound existingUpperBound : existingUpperBounds) { - String releaseBranch = existingUpperBound.releaseBranch(); + String upperBoundName = existingUpperBound.name(); - if (targetReleaseBranches.contains(releaseBranch)) { - // Case: targeting this branch, find an existing id for this branch if it exists + if (targetUpperBoundNames.contains(upperBoundName)) { + // Case: targeting this branch, find an existing definitionId for this branch if it exists TransportVersionId targetId = maybeGetExistingId(existingUpperBound, existingDefinition, name); if (targetId == null) { - // Case: an id doesn't yet exist for this branch, so create one - int increment = releaseBranch.equals(mainReleaseBranch) ? primaryIncrement : 1; - targetId = TransportVersionId.fromInt(existingUpperBound.id().complete() + increment); - var newUpperBound = new TransportVersionUpperBound(releaseBranch, name, targetId); + // Case: an definitionId doesn't yet exist for this branch, so create one + int increment = upperBoundName.equals(currentUpperBoundName) ? primaryIncrement : 1; + targetId = TransportVersionId.fromInt(existingUpperBound.definitionId().complete() + increment); + var newUpperBound = new TransportVersionUpperBound(upperBoundName, name, targetId); resources.writeUpperBound(newUpperBound); } ids.add(targetId); @@ -148,23 +131,23 @@ private List updateUpperBounds( return ids; } - private String getTargetName(TransportVersionResourcesService resources, Set referencedNames, List changedDefinitions) { + private String getTargetDefinitionName(TransportVersionResourcesService resources, Set referencedNames, List changedDefinitions) { if (getTransportVersionName().isPresent()) { - // an explicit name was passed in, so use it + // an explicit definitionName was passed in, so use it return getTransportVersionName().get(); } // First check for unreferenced names. We only care about the first one. If there is more than one // validation will fail later and the developer will have to remove one. When that happens, generation - // will re-run and we will fixup the state to use whatever new name remains. + // will re-run and we will fixup the state to use whatever new definitionName remains. for (String referencedName : referencedNames) { if (resources.referableDefinitionExists(referencedName) == false) { return referencedName; } } - // Since we didn't find any missing names, we use the first changed name. If there is more than - // one changed name, validation will fail later, just as above. + // Since we didn't find any missing names, we use the first changed definitionName. If there is more than + // one changed definitionName, validation will fail later, just as above. if (changedDefinitions.isEmpty()) { return ""; } else { @@ -172,19 +155,33 @@ private String getTargetName(TransportVersionResourcesService resources, Set getTargetReleaseBranches() { - Set releaseBranches = new HashSet<>(); - releaseBranches.add(getMainReleaseBranch().get()); + private Set getTargetUpperBoundNames(List mainUpperBounds) { + Set targetUpperBoundNames = new HashSet<>(); + targetUpperBoundNames.add(getCurrentUpperBoundName().get()); if (getBackportBranches().isPresent()) { - releaseBranches.addAll(List.of(getBackportBranches().get().split(","))); + targetUpperBoundNames.addAll(List.of(getBackportBranches().get().split(","))); + } + + Set missingBranches = new HashSet<>(targetUpperBoundNames); + List knownUpperBoundNames = new ArrayList<>(); + for (TransportVersionUpperBound upperBound : mainUpperBounds) { + knownUpperBoundNames.add(upperBound.name()); + missingBranches.remove(upperBound.name()); + } + if (missingBranches.isEmpty() == false) { + List sortedMissing = missingBranches.stream().sorted().toList(); + List sortedKnown = knownUpperBoundNames.stream().sorted().toList(); + throw new IllegalArgumentException( + "Missing upper bounds files for branches " + sortedMissing + ", known branches are " + sortedKnown + ); } - return releaseBranches; + return targetUpperBoundNames; } private void resetAllUpperBounds(TransportVersionResourcesService resources) throws IOException { @@ -214,24 +211,24 @@ private TransportVersionId maybeGetExistingId( String name ) { if (existingDefinition == null) { - // the name doesn't yet exist, so there is no id to return + // the definitionName doesn't yet exist, so there is no definitionId to return return null; } - if (upperBound.name().equals(name)) { - // the name exists and this upper bound already points at it - return upperBound.id(); + if (upperBound.definitionName().equals(name)) { + // the definitionName exists and this upper bound already points at it + return upperBound.definitionId(); } - if (upperBound.releaseBranch().equals(getMainReleaseBranch().get())) { - // the upper bound is for main, so return the primary id + if (upperBound.name().equals(getCurrentUpperBoundName().get())) { + // the upper bound is for main, so return the primary definitionId return existingDefinition.ids().get(0); } - // the upper bound is for a non-main branch, so find the id with the same base + // the upper bound is for a non-main branch, so find the definitionId with the same base for (TransportVersionId id : existingDefinition.ids()) { - if (id.base() == upperBound.id().base()) { + if (id.base() == upperBound.definitionId().base()) { return id; } } - return null; // no id for this release branch + return null; // no definitionId for this release branch } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionDefinition.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionDefinition.java index 65f4caeb95206..4294f5e4e440a 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionDefinition.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionDefinition.java @@ -25,7 +25,7 @@ public static TransportVersionDefinition fromString(Path file, String contents) try { ids.add(TransportVersionId.fromString(rawId)); } catch (NumberFormatException e) { - throw new IllegalStateException("Failed to parse id " + rawId + " in " + file, e); + throw new IllegalStateException("Failed to parse definitionId " + rawId + " in " + file, e); } } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java index 814722f852cb6..11834e73d9bbc 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java @@ -80,7 +80,7 @@ public void apply(Project project) { t.getReferencesFiles().setFrom(tvReferencesConfig); t.getPrimaryIncrement().convention(1000); Version esVersion = VersionProperties.getElasticsearchVersion(); - t.getMainReleaseBranch().convention(esVersion.getMajor() + "." + esVersion.getMinor()); + t.getCurrentUpperBoundName().convention(esVersion.getMajor() + "." + esVersion.getMinor()); }); validateTask.configure(t -> t.mustRunAfter(generateDefinitionsTask)); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java index 01aec6e677f42..2d91d3fd2c565 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java @@ -48,7 +48,7 @@ *

  • /transport/definitions/unreferable/ * - Definitions which contain ids that are known at runtime, but cannot be looked up by name.
    • *
  • /transport/upper_bounds/ - * - The maximum transport version definition that will be loaded for each release releaseBranch.
    • + * - The maximum transport version definition that will be loaded on a branch. * */ public abstract class TransportVersionResourcesService implements BuildService { @@ -173,22 +173,22 @@ Path getUnreferableDefinitionRepositoryPath(TransportVersionDefinition definitio return rootDir.relativize(transportResourcesDir.resolve(getUnreferableDefinitionRelativePath(definition.name()))); } - /** Read all upper bound files and return them mapped by their release releaseBranch */ + /** Read all upper bound files and return them mapped by their release name */ Map getUpperBounds() throws IOException { Map upperBounds = new HashMap<>(); try (var stream = Files.list(transportResourcesDir.resolve(UPPER_BOUNDS_DIR))) { for (var latestFile : stream.toList()) { String contents = Files.readString(latestFile, StandardCharsets.UTF_8).strip(); var upperBound = TransportVersionUpperBound.fromString(latestFile, contents); - upperBounds.put(upperBound.releaseBranch(), upperBound); + upperBounds.put(upperBound.name(), upperBound); } } return upperBounds; } - /** Retrieve the upper bound for the given release releaseBranch on main */ - TransportVersionUpperBound getUpperBoundFromMain(String releaseBranch) { - Path resourcePath = getUpperBoundRelativePath(releaseBranch); + /** Retrieve the upper bound for the given release name on main */ + TransportVersionUpperBound getUpperBoundFromMain(String name) { + Path resourcePath = getUpperBoundRelativePath(name); return getMainFile(resourcePath, TransportVersionUpperBound::fromString); } @@ -208,18 +208,18 @@ List getUpperBoundsFromMain() throws IOException { /** Write the given upper bound to a file in the transport resources */ void writeUpperBound(TransportVersionUpperBound upperBound) throws IOException { - Path path = transportResourcesDir.resolve(getUpperBoundRelativePath(upperBound.releaseBranch())); + Path path = transportResourcesDir.resolve(getUpperBoundRelativePath(upperBound.name())); logger.debug("Writing upper bound [" + upperBound + "] to [" + path + "]"); - Files.writeString(path, upperBound.name() + "," + upperBound.id().complete() + "\n", StandardCharsets.UTF_8); + Files.writeString(path, upperBound.definitionName() + "," + upperBound.definitionId().complete() + "\n", StandardCharsets.UTF_8); } /** Return the path within the repository of the given latest */ Path getUpperBoundRepositoryPath(TransportVersionUpperBound latest) { - return rootDir.relativize(transportResourcesDir.resolve(getUpperBoundRelativePath(latest.releaseBranch()))); + return rootDir.relativize(transportResourcesDir.resolve(getUpperBoundRelativePath(latest.name()))); } - private Path getUpperBoundRelativePath(String releaseBranch) { - return UPPER_BOUNDS_DIR.resolve(releaseBranch + ".csv"); + private Path getUpperBoundRelativePath(String name) { + return UPPER_BOUNDS_DIR.resolve(name + ".csv"); } // Return the transport version resources paths that exist in main @@ -258,7 +258,7 @@ private Set getChangedResources() { return changedResources.get(); } - // Read a transport version resource from the main releaseBranch, or return null if it doesn't exist on main + // Read a transport version resource from the main name, or return null if it doesn't exist on main private T getMainFile(Path resourcePath, BiFunction parser) { String pathString = resourcePath.toString().replace('\\', '/'); // normalize to forward slash that git uses if (getMainResources().contains(pathString) == false) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUpperBound.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUpperBound.java index f675d7db67a20..d3b914647f0fa 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUpperBound.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUpperBound.java @@ -14,9 +14,9 @@ /** * An object to represent the loaded version of a transport version upper bound. * - * An upper bound is the maximum transport version id that should be loaded for a given release branch. + * An upper bound is the maximum transport version definitionId that should be loaded for a given release branch. */ -record TransportVersionUpperBound(String releaseBranch, String name, TransportVersionId id) { +record TransportVersionUpperBound(String name, String definitionName, TransportVersionId definitionId) { public static TransportVersionUpperBound fromString(Path file, String contents) { String filename = file.getFileName().toString(); assert filename.endsWith(".csv"); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java index 97820aab29d82..6c03e1dbe5315 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java @@ -105,7 +105,7 @@ private Map collectAllDefinitions( TransportVersionDefinition existing = allDefinitions.put(entry.getKey(), entry.getValue()); if (existing != null) { Path unreferablePath = getResources().get().getUnreferableDefinitionRepositoryPath(entry.getValue()); - throwDefinitionFailure(existing, "has same name as unreferable definition [" + unreferablePath + "]"); + throwDefinitionFailure(existing, "has same definitionName as unreferable definition [" + unreferablePath + "]"); } } return allDefinitions; @@ -123,7 +123,7 @@ private Map> collectIdsByBase(Collection a.id().complete())); } @@ -144,7 +144,7 @@ private void validateNamedDefinition(TransportVersionDefinition definition, Set< throwDefinitionFailure(definition, "is not referenced"); } if (NAME_FORMAT.matcher(definition.name()).matches() == false) { - throwDefinitionFailure(definition, "does not have a valid name, must be lowercase alphanumeric and underscore"); + throwDefinitionFailure(definition, "does not have a valid definitionName, must be lowercase alphanumeric and underscore"); } if (definition.ids().isEmpty()) { throwDefinitionFailure(definition, "does not contain any ids"); @@ -157,18 +157,18 @@ private void validateNamedDefinition(TransportVersionDefinition definition, Set< if (ndx == 0) { if (getShouldValidatePrimaryIdNotPatch().get() && id.patch() != 0) { - throwDefinitionFailure(definition, "has patch version " + id.complete() + " as primary id"); + throwDefinitionFailure(definition, "has patch version " + id.complete() + " as primary definitionId"); } } else { if (id.patch() == 0) { - throwDefinitionFailure(definition, "contains bwc id [" + id + "] with a patch part of 0"); + throwDefinitionFailure(definition, "contains bwc definitionId [" + id + "] with a patch part of 0"); } } - // check modifications of ids on same releaseBranch, ie sharing same base + // check modifications of ids on same name, ie sharing same base TransportVersionId maybeModifiedId = existingIdsByBase.get(id.base()); if (maybeModifiedId != null && maybeModifiedId.complete() != id.complete()) { - throwDefinitionFailure(definition, "modifies existing patch id from " + maybeModifiedId + " to " + id); + throwDefinitionFailure(definition, "modifies existing patch definitionId from " + maybeModifiedId + " to " + id); } } } @@ -182,9 +182,9 @@ private void validateUnreferableDefinition(TransportVersionDefinition definition throwDefinitionFailure(definition, "does not contain any ids"); } if (definition.ids().size() > 1) { - throwDefinitionFailure(definition, " contains more than one id"); + throwDefinitionFailure(definition, " contains more than one definitionId"); } - // note: no name validation, anything that is a valid filename is ok, this allows eg initial_8.9.1 + // note: no definitionName validation, anything that is a valid filename is ok, this allows eg initial_8.9.1 } private void validateIdenticalPrimaryId(TransportVersionDefinition definition, TransportVersionDefinition originalDefinition) { @@ -193,7 +193,7 @@ private void validateIdenticalPrimaryId(TransportVersionDefinition definition, T int primaryId = definition.ids().get(0).complete(); int originalPrimaryId = originalDefinition.ids().get(0).complete(); if (primaryId != originalPrimaryId) { - throwDefinitionFailure(definition, "has modified primary id from " + originalPrimaryId + " to " + primaryId); + throwDefinitionFailure(definition, "has modified primary definitionId from " + originalPrimaryId + " to " + primaryId); } } @@ -202,27 +202,27 @@ private void validateUpperBound( Map definitions, Map> idsByBase ) { - TransportVersionDefinition upperBoundDefinition = definitions.get(upperBound.name()); + TransportVersionDefinition upperBoundDefinition = definitions.get(upperBound.definitionName()); if (upperBoundDefinition == null) { - throwUpperBoundFailure(upperBound, "contains transport version name [" + upperBound.name() + "] which is not defined"); + throwUpperBoundFailure(upperBound, "contains transport version definitionName [" + upperBound.definitionName() + "] which is not defined"); } - if (upperBoundDefinition.ids().contains(upperBound.id()) == false) { + if (upperBoundDefinition.ids().contains(upperBound.definitionId()) == false) { Path relativePath = getResources().get().getReferableDefinitionRepositoryPath(upperBoundDefinition); - throwUpperBoundFailure(upperBound, "has id " + upperBound.id() + " which is not in definition [" + relativePath + "]"); + throwUpperBoundFailure(upperBound, "has definitionId " + upperBound.definitionId() + " which is not in definition [" + relativePath + "]"); } - List baseIds = idsByBase.get(upperBound.id().base()); + List baseIds = idsByBase.get(upperBound.definitionId().base()); IdAndDefinition lastId = baseIds.getLast(); - if (lastId.id().complete() != upperBound.id().complete()) { + if (lastId.id().complete() != upperBound.definitionId().complete()) { throwUpperBoundFailure( upperBound, - "has id " - + upperBound.id() + "has definitionId " + + upperBound.definitionId() + " from [" - + upperBound.name() + + upperBound.definitionName() + "] with base " - + upperBound.id().base() - + " but another id " + + upperBound.definitionId().base() + + " but another definitionId " + lastId.id().complete() + " from [" + lastId.definition().name() @@ -230,12 +230,12 @@ private void validateUpperBound( ); } - TransportVersionUpperBound existingUpperBound = getResources().get().getUpperBoundFromMain(upperBound.releaseBranch()); + TransportVersionUpperBound existingUpperBound = getResources().get().getUpperBoundFromMain(upperBound.name()); if (existingUpperBound != null) { - if (upperBound.id().patch() != 0 && upperBound.id().base() != existingUpperBound.id().base()) { + if (upperBound.definitionId().patch() != 0 && upperBound.definitionId().base() != existingUpperBound.definitionId().base()) { throwUpperBoundFailure( upperBound, - "modifies base id from " + existingUpperBound.id().base() + " to " + upperBound.id().base() + "modifies base definitionId from " + existingUpperBound.definitionId().base() + " to " + upperBound.definitionId().base() ); } } @@ -251,13 +251,13 @@ private void validateBase(int base, List ids) { Path existingDefinitionPath = getResources().get().getReferableDefinitionRepositoryPath(previous.definition); throwDefinitionFailure( current.definition(), - "contains id " + current.id + " already defined in [" + existingDefinitionPath + "]" + "contains definitionId " + current.id + " already defined in [" + existingDefinitionPath + "]" ); } if (getShouldValidateDensity().get() && previous.id().complete() - 1 != current.id().complete()) { throw new IllegalStateException( - "Transport version base id " + base + " is missing patch ids between " + current.id() + " and " + previous.id() + "Transport version base definitionId " + base + " is missing patch ids between " + current.id() + " and " + previous.id() ); } previous = current; @@ -268,20 +268,20 @@ private void validateLargestIdIsUsed( Map upperBounds, Map allDefinitions ) { - // first id is always the highest within a definition, and validated earlier - // note we use min instead of max because the id comparator is in descending order + // first definitionId is always the highest within a definition, and validated earlier + // note we use min instead of max because the definitionId comparator is in descending order var highestDefinition = allDefinitions.values().stream().min(Comparator.comparing(d -> d.ids().get(0))).get(); var highestId = highestDefinition.ids().get(0); for (var upperBound : upperBounds.values()) { - if (upperBound.id().equals(highestId)) { + if (upperBound.definitionId().equals(highestId)) { return; } } throwDefinitionFailure( highestDefinition, - "has the highest transport version id [" + highestId + "] but is not present in any upper bounds files" + "has the highest transport version definitionId [" + highestId + "] but is not present in any upper bounds files" ); } From 454400a2b4141dcd8742978cff5b038bce15c5ff Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 4 Sep 2025 14:13:20 -0700 Subject: [PATCH 62/68] more renames --- ...enerateTransportVersionDefinitionTask.java | 37 +++++++++---------- .../TransportVersionResourcesPlugin.java | 2 +- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index fafdebb7bea05..c3532486de4c8 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -61,8 +61,8 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask @Input @Optional - @Option(option = "increment", description = "The amount to increment the primary definitionId for the main name") - public abstract Property getPrimaryIncrement(); + @Option(option = "increment", description = "The amount to increment the id from the current upper bounds file by") + public abstract Property getIncrement(); /** * The name of the upper bounds file which will be used at runtime on the current branch. Normally @@ -97,27 +97,27 @@ private List updateUpperBounds( TransportVersionResourcesService resources, List existingUpperBounds, Set targetUpperBoundNames, - String name + String definitionName ) throws IOException { String currentUpperBoundName = getCurrentUpperBoundName().get(); - int primaryIncrement = getPrimaryIncrement().get(); - if (primaryIncrement <= 0) { - throw new IllegalArgumentException("Invalid increment " + primaryIncrement + ", must be a positive integer"); + int increment = getIncrement().get(); + if (increment <= 0) { + throw new IllegalArgumentException("Invalid increment " + increment + ", must be a positive integer"); } List ids = new ArrayList<>(); - TransportVersionDefinition existingDefinition = resources.getReferableDefinitionFromMain(name); + TransportVersionDefinition existingDefinition = resources.getReferableDefinitionFromMain(definitionName); for (TransportVersionUpperBound existingUpperBound : existingUpperBounds) { String upperBoundName = existingUpperBound.name(); if (targetUpperBoundNames.contains(upperBoundName)) { - // Case: targeting this branch, find an existing definitionId for this branch if it exists - TransportVersionId targetId = maybeGetExistingId(existingUpperBound, existingDefinition, name); + // Case: targeting this upper bound, find an existing id if it exists + TransportVersionId targetId = maybeGetExistingId(existingUpperBound, existingDefinition, definitionName); if (targetId == null) { - // Case: an definitionId doesn't yet exist for this branch, so create one - int increment = upperBoundName.equals(currentUpperBoundName) ? primaryIncrement : 1; - targetId = TransportVersionId.fromInt(existingUpperBound.definitionId().complete() + increment); - var newUpperBound = new TransportVersionUpperBound(upperBoundName, name, targetId); + // Case: an id doesn't yet exist for this upper bound, so create one + int targetIncrement = upperBoundName.equals(currentUpperBoundName) ? increment : 1; + targetId = TransportVersionId.fromInt(existingUpperBound.definitionId().complete() + targetIncrement); + var newUpperBound = new TransportVersionUpperBound(upperBoundName, definitionName, targetId); resources.writeUpperBound(newUpperBound); } ids.add(targetId); @@ -185,7 +185,6 @@ private Set getTargetUpperBoundNames(List ma } private void resetAllUpperBounds(TransportVersionResourcesService resources) throws IOException { - // TODO: this should also _delete_ extraneous files from latest? for (TransportVersionUpperBound upperBound : resources.getUpperBoundsFromMain()) { resources.writeUpperBound(upperBound); } @@ -211,7 +210,7 @@ private TransportVersionId maybeGetExistingId( String name ) { if (existingDefinition == null) { - // the definitionName doesn't yet exist, so there is no definitionId to return + // the definitionName doesn't yet exist, so there is no id to return return null; } if (upperBound.definitionName().equals(name)) { @@ -219,16 +218,16 @@ private TransportVersionId maybeGetExistingId( return upperBound.definitionId(); } if (upperBound.name().equals(getCurrentUpperBoundName().get())) { - // the upper bound is for main, so return the primary definitionId - return existingDefinition.ids().get(0); + // this is the upper bound of the current branch, so use the primary id + return existingDefinition.ids().getFirst(); } - // the upper bound is for a non-main branch, so find the definitionId with the same base + // the upper bound is for a non-current branch, so find the id with the same base for (TransportVersionId id : existingDefinition.ids()) { if (id.base() == upperBound.definitionId().base()) { return id; } } - return null; // no definitionId for this release branch + return null; // no existing id for this upper bound } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java index 11834e73d9bbc..1125483f4d471 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java @@ -78,7 +78,7 @@ public void apply(Project project) { t.setGroup(taskGroup); t.setDescription("(Re)generates a transport version definition file"); t.getReferencesFiles().setFrom(tvReferencesConfig); - t.getPrimaryIncrement().convention(1000); + t.getIncrement().convention(1000); Version esVersion = VersionProperties.getElasticsearchVersion(); t.getCurrentUpperBoundName().convention(esVersion.getMajor() + "." + esVersion.getMinor()); }); From bd8f430c875e671674114dd4677ac03400b7a392 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 4 Sep 2025 15:43:02 -0700 Subject: [PATCH 63/68] use upstream instead of main --- ...enerateTransportVersionDefinitionTask.java | 25 +++-- .../TransportVersionResourcesService.java | 101 ++++++++++++------ ...ValidateTransportVersionResourcesTask.java | 6 +- 3 files changed, 88 insertions(+), 44 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index c3532486de4c8..214496bcab02a 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -33,6 +33,9 @@ * are runtime resources that TransportVersion loads statically. * They contain a comma separated list of integer ids. Each file is named the same * as the transport version definitionName itself (with the .csv suffix). + * + * Additionally, when definition files are added or updated, the upper bounds files + * for each relevant branch's upper bound file are also updated. */ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask { @@ -48,8 +51,8 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask @Input @Optional - @Option(option = "name", description = "The definitionName of the Transport Version definition, e.g. --definitionName=my_new_tv") - public abstract Property getTransportVersionName(); + @Option(option = "name", description = "The name of the Transport Version definition, e.g. --name=my_new_tv") + public abstract Property getDefinitionName(); @Input @Optional @@ -78,14 +81,14 @@ public void run() throws IOException { List changedDefinitionNames = resources.getChangedReferableDefinitionNames(); String targetDefinitionName = getTargetDefinitionName(resources, referencedNames, changedDefinitionNames); - List mainUpperBounds = resources.getUpperBoundsFromMain(); - Set targetUpperBoundNames = getTargetUpperBoundNames(mainUpperBounds); + List upstreamUpperBounds = resources.getUpperBoundsFromUpstream(); + Set targetUpperBoundNames = getTargetUpperBoundNames(upstreamUpperBounds); getLogger().lifecycle("Generating transport version name: " + targetDefinitionName); if (targetDefinitionName.isEmpty()) { resetAllUpperBounds(resources); } else { - List ids = updateUpperBounds(resources, mainUpperBounds, targetUpperBoundNames, targetDefinitionName); + List ids = updateUpperBounds(resources, upstreamUpperBounds, targetUpperBoundNames, targetDefinitionName); // (Re)write the definition file. resources.writeReferableDefinition(new TransportVersionDefinition(targetDefinitionName, ids)); } @@ -106,7 +109,7 @@ private List updateUpperBounds( } List ids = new ArrayList<>(); - TransportVersionDefinition existingDefinition = resources.getReferableDefinitionFromMain(definitionName); + TransportVersionDefinition existingDefinition = resources.getReferableDefinitionFromUpstream(definitionName); for (TransportVersionUpperBound existingUpperBound : existingUpperBounds) { String upperBoundName = existingUpperBound.name(); @@ -132,9 +135,9 @@ private List updateUpperBounds( } private String getTargetDefinitionName(TransportVersionResourcesService resources, Set referencedNames, List changedDefinitions) { - if (getTransportVersionName().isPresent()) { + if (getDefinitionName().isPresent()) { // an explicit definitionName was passed in, so use it - return getTransportVersionName().get(); + return getDefinitionName().get(); } // First check for unreferenced names. We only care about the first one. If there is more than one @@ -160,7 +163,7 @@ private String getTargetDefinitionName(TransportVersionResourcesService resource } } - private Set getTargetUpperBoundNames(List mainUpperBounds) { + private Set getTargetUpperBoundNames(List upstreamUpperBounds) { Set targetUpperBoundNames = new HashSet<>(); targetUpperBoundNames.add(getCurrentUpperBoundName().get()); if (getBackportBranches().isPresent()) { @@ -169,7 +172,7 @@ private Set getTargetUpperBoundNames(List ma Set missingBranches = new HashSet<>(targetUpperBoundNames); List knownUpperBoundNames = new ArrayList<>(); - for (TransportVersionUpperBound upperBound : mainUpperBounds) { + for (TransportVersionUpperBound upperBound : upstreamUpperBounds) { knownUpperBoundNames.add(upperBound.name()); missingBranches.remove(upperBound.name()); } @@ -185,7 +188,7 @@ private Set getTargetUpperBoundNames(List ma } private void resetAllUpperBounds(TransportVersionResourcesService resources) throws IOException { - for (TransportVersionUpperBound upperBound : resources.getUpperBoundsFromMain()) { + for (TransportVersionUpperBound upperBound : resources.getUpperBoundsFromUpstream()) { resources.writeUpperBound(upperBound); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java index 2d91d3fd2c565..1ba0a5bf42199 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java @@ -71,7 +71,8 @@ public interface Parameters extends BuildServiceParameters { private final Path transportResourcesDir; private final Path rootDir; - private final AtomicReference> mainResources = new AtomicReference<>(null); + private final AtomicReference upstreamRefName = new AtomicReference<>(); + private final AtomicReference> upstreamResources = new AtomicReference<>(null); private final AtomicReference> changedResources = new AtomicReference<>(null); @Inject @@ -106,13 +107,13 @@ Map getReferableDefinitions() throws IOExcep return readDefinitions(transportResourcesDir.resolve(REFERABLE_DIR)); } - /** Get a referable definition from main if it exists there, or null otherwise */ - TransportVersionDefinition getReferableDefinitionFromMain(String name) { + /** Get a referable definition from upstream if it exists there, or null otherwise */ + TransportVersionDefinition getReferableDefinitionFromUpstream(String name) { Path resourcePath = getReferableDefinitionRelativePath(name); - return getMainFile(resourcePath, TransportVersionDefinition::fromString); + return getUpstreamFile(resourcePath, TransportVersionDefinition::fromString); } - /** Get the definition names which have local changes relative to the main branch */ + /** Get the definition names which have local changes relative to upstream */ List getChangedReferableDefinitionNames() { List changedDefinitions = new ArrayList<>(); String referablePrefix = REFERABLE_DIR.toString(); @@ -162,10 +163,10 @@ Map getUnreferableDefinitions() throws IOExc return readDefinitions(transportResourcesDir.resolve(UNREFERABLE_DIR)); } - /** Get a referable definition from main if it exists there, or null otherwise */ - TransportVersionDefinition getUnreferableDefinitionFromMain(String name) { + /** Get a referable definition from upstream if it exists there, or null otherwise */ + TransportVersionDefinition getUnreferableDefinitionFromUpstream(String name) { Path resourcePath = getUnreferableDefinitionRelativePath(name); - return getMainFile(resourcePath, TransportVersionDefinition::fromString); + return getUpstreamFile(resourcePath, TransportVersionDefinition::fromString); } /** Return the path within the repository of the given referable definition */ @@ -186,21 +187,21 @@ Map getUpperBounds() throws IOException { return upperBounds; } - /** Retrieve the upper bound for the given release name on main */ - TransportVersionUpperBound getUpperBoundFromMain(String name) { + /** Retrieve an upper bound from upstream by name */ + TransportVersionUpperBound getUpperBoundFromUpstream(String name) { Path resourcePath = getUpperBoundRelativePath(name); - return getMainFile(resourcePath, TransportVersionUpperBound::fromString); + return getUpstreamFile(resourcePath, TransportVersionUpperBound::fromString); } - /** Retrieve all upper bounds that exist on main */ - List getUpperBoundsFromMain() throws IOException { + /** Retrieve all upper bounds that exist in upstream */ + List getUpperBoundsFromUpstream() throws IOException { List upperBounds = new ArrayList<>(); - for (String mainPathString : getMainResources()) { - Path mainPath = Path.of(mainPathString); - if (mainPath.startsWith(UPPER_BOUNDS_DIR) == false) { + for (String upstreamPathString : getUpstreamResources()) { + Path upstreamPath = Path.of(upstreamPathString); + if (upstreamPath.startsWith(UPPER_BOUNDS_DIR) == false) { continue; } - TransportVersionUpperBound upperBound = getMainFile(mainPath, TransportVersionUpperBound::fromString); + TransportVersionUpperBound upperBound = getUpstreamFile(upstreamPath, TransportVersionUpperBound::fromString); upperBounds.add(upperBound); } return upperBounds; @@ -222,27 +223,67 @@ private Path getUpperBoundRelativePath(String name) { return UPPER_BOUNDS_DIR.resolve(name + ".csv"); } - // Return the transport version resources paths that exist in main - private Set getMainResources() { - if (mainResources.get() == null) { - synchronized (mainResources) { - String output = gitCommand("ls-tree", "--name-only", "-r", "main", "."); + private String getUpstreamRefName() { + if (upstreamRefName.get() == null) { + synchronized (upstreamRefName) { + String remotesOutput = gitCommand("remote").strip(); + + String refName; + if (remotesOutput.isEmpty()) { + refName = "main"; // fallback to local main if no remotes, this happens in tests + } else { + List remoteNames = List.of(remotesOutput.split("\n")); + String transportVersionRemoteName = "transport-version-resources-upstream"; + if (remoteNames.contains(transportVersionRemoteName) == false) { + // our special remote doesn't exist yet, so create it + String upstreamUrl = null; + for (String remoteName : remoteNames) { + String getUrlOutput = gitCommand("remote", "get-url", remoteName).strip(); + if (getUrlOutput.startsWith("git@github.com:elastic/") || getUrlOutput.startsWith("https://github.com/elastic/")) { + upstreamUrl = getUrlOutput; + } + } + + if (upstreamUrl != null) { + gitCommand("remote", "add", transportVersionRemoteName, upstreamUrl); + } else { + throw new RuntimeException("No elastic github remotes found to copy"); + } + } + + // make sure the remote main ref is up to date + gitCommand("fetch", transportVersionRemoteName, "main"); + + refName = transportVersionRemoteName + "/main"; + } + upstreamRefName.set(refName); + + } + } + return upstreamRefName.get(); + } + + // Return the transport version resources paths that exist in upstream + private Set getUpstreamResources() { + if (upstreamResources.get() == null) { + synchronized (upstreamResources) { + String output = gitCommand("ls-tree", "--name-only", "-r", getUpstreamRefName(), "."); HashSet resources = new HashSet<>(); Collections.addAll(resources, output.split("\n")); // git always outputs LF - mainResources.set(resources); + upstreamResources.set(resources); } } - return mainResources.get(); + return upstreamResources.get(); } - // Return the transport version resources paths that have been changed relative to main + // Return the transport version resources paths that have been changed relative to upstream private Set getChangedResources() { if (changedResources.get() == null) { synchronized (changedResources) { HashSet resources = new HashSet<>(); - String diffOutput = gitCommand("diff", "--name-only", "main", "."); + String diffOutput = gitCommand("diff", "--name-only", getUpstreamRefName(), "."); if (diffOutput.strip().isEmpty() == false) { Collections.addAll(resources, diffOutput.split("\n")); // git always outputs LF } @@ -258,14 +299,14 @@ private Set getChangedResources() { return changedResources.get(); } - // Read a transport version resource from the main name, or return null if it doesn't exist on main - private T getMainFile(Path resourcePath, BiFunction parser) { + // Read a transport version resource from the upstream, or return null if it doesn't exist there + private T getUpstreamFile(Path resourcePath, BiFunction parser) { String pathString = resourcePath.toString().replace('\\', '/'); // normalize to forward slash that git uses - if (getMainResources().contains(pathString) == false) { + if (getUpstreamResources().contains(pathString) == false) { return null; } - String content = gitCommand("show", "main:./" + pathString).strip(); + String content = gitCommand("show", getUpstreamRefName() + ":./" + pathString).strip(); return parser.apply(resourcePath, content); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java index 6c03e1dbe5315..ffa3b77655297 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java @@ -134,7 +134,7 @@ private void validateNamedDefinition(TransportVersionDefinition definition, Set< // validate any modifications Map existingIdsByBase = new HashMap<>(); - TransportVersionDefinition originalDefinition = getResources().get().getReferableDefinitionFromMain(definition.name()); + TransportVersionDefinition originalDefinition = getResources().get().getReferableDefinitionFromUpstream(definition.name()); if (originalDefinition != null) { validateIdenticalPrimaryId(definition, originalDefinition); originalDefinition.ids().forEach(id -> existingIdsByBase.put(id.base(), id)); @@ -174,7 +174,7 @@ private void validateNamedDefinition(TransportVersionDefinition definition, Set< } private void validateUnreferableDefinition(TransportVersionDefinition definition) { - TransportVersionDefinition originalDefinition = getResources().get().getUnreferableDefinitionFromMain(definition.name()); + TransportVersionDefinition originalDefinition = getResources().get().getUnreferableDefinitionFromUpstream(definition.name()); if (originalDefinition != null) { validateIdenticalPrimaryId(definition, originalDefinition); } @@ -230,7 +230,7 @@ private void validateUpperBound( ); } - TransportVersionUpperBound existingUpperBound = getResources().get().getUpperBoundFromMain(upperBound.name()); + TransportVersionUpperBound existingUpperBound = getResources().get().getUpperBoundFromUpstream(upperBound.name()); if (existingUpperBound != null) { if (upperBound.definitionId().patch() != 0 && upperBound.definitionId().base() != existingUpperBound.definitionId().base()) { throwUpperBoundFailure( From 94df820cbf970f18479cb5bde7044b202caff682 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 4 Sep 2025 15:44:12 -0700 Subject: [PATCH 64/68] iter --- ...enerateTransportVersionDefinitionTask.java | 8 +++++-- .../TransportVersionResourcesService.java | 3 ++- ...ValidateTransportVersionResourcesTask.java | 22 +++++++++++++++---- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index 214496bcab02a..a4918f79bc6ae 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -134,7 +134,11 @@ private List updateUpperBounds( return ids; } - private String getTargetDefinitionName(TransportVersionResourcesService resources, Set referencedNames, List changedDefinitions) { + private String getTargetDefinitionName( + TransportVersionResourcesService resources, + Set referencedNames, + List changedDefinitions + ) { if (getDefinitionName().isPresent()) { // an explicit definitionName was passed in, so use it return getDefinitionName().get(); @@ -180,7 +184,7 @@ private Set getTargetUpperBoundNames(List up List sortedMissing = missingBranches.stream().sorted().toList(); List sortedKnown = knownUpperBoundNames.stream().sorted().toList(); throw new IllegalArgumentException( - "Missing upper bounds files for branches " + sortedMissing + ", known branches are " + sortedKnown + "Missing upper bounds files for branches " + sortedMissing + ", known branches are " + sortedKnown ); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java index 1ba0a5bf42199..85590cb3cb196 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java @@ -239,7 +239,8 @@ private String getUpstreamRefName() { String upstreamUrl = null; for (String remoteName : remoteNames) { String getUrlOutput = gitCommand("remote", "get-url", remoteName).strip(); - if (getUrlOutput.startsWith("git@github.com:elastic/") || getUrlOutput.startsWith("https://github.com/elastic/")) { + if (getUrlOutput.startsWith("git@github.com:elastic/") + || getUrlOutput.startsWith("https://github.com/elastic/")) { upstreamUrl = getUrlOutput; } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java index ffa3b77655297..f8d5dba73ee4b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java @@ -204,11 +204,17 @@ private void validateUpperBound( ) { TransportVersionDefinition upperBoundDefinition = definitions.get(upperBound.definitionName()); if (upperBoundDefinition == null) { - throwUpperBoundFailure(upperBound, "contains transport version definitionName [" + upperBound.definitionName() + "] which is not defined"); + throwUpperBoundFailure( + upperBound, + "contains transport version definitionName [" + upperBound.definitionName() + "] which is not defined" + ); } if (upperBoundDefinition.ids().contains(upperBound.definitionId()) == false) { Path relativePath = getResources().get().getReferableDefinitionRepositoryPath(upperBoundDefinition); - throwUpperBoundFailure(upperBound, "has definitionId " + upperBound.definitionId() + " which is not in definition [" + relativePath + "]"); + throwUpperBoundFailure( + upperBound, + "has definitionId " + upperBound.definitionId() + " which is not in definition [" + relativePath + "]" + ); } List baseIds = idsByBase.get(upperBound.definitionId().base()); @@ -235,7 +241,10 @@ private void validateUpperBound( if (upperBound.definitionId().patch() != 0 && upperBound.definitionId().base() != existingUpperBound.definitionId().base()) { throwUpperBoundFailure( upperBound, - "modifies base definitionId from " + existingUpperBound.definitionId().base() + " to " + upperBound.definitionId().base() + "modifies base definitionId from " + + existingUpperBound.definitionId().base() + + " to " + + upperBound.definitionId().base() ); } } @@ -257,7 +266,12 @@ private void validateBase(int base, List ids) { if (getShouldValidateDensity().get() && previous.id().complete() - 1 != current.id().complete()) { throw new IllegalStateException( - "Transport version base definitionId " + base + " is missing patch ids between " + current.id() + " and " + previous.id() + "Transport version base definitionId " + + base + + " is missing patch ids between " + + current.id() + + " and " + + previous.id() ); } previous = current; From bfa290d5c1d5e64298571dcceba52d55ce576930 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 4 Sep 2025 17:13:32 -0700 Subject: [PATCH 65/68] test fixes --- .../TransportVersionValidationFuncTest.groovy | 24 +++++++-------- .../transport/TransportVersionDefinition.java | 2 +- ...ValidateTransportVersionResourcesTask.java | 30 +++++++++---------- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy index 3772eb1373fd3..989954a5407e8 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy @@ -87,7 +87,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes def result = validateResourcesFails() then: assertValidateResourcesFailure(result, "Transport version definition file " + - "[myserver/src/main/resources/transport/definitions/referable/existing_92.csv] contains definitionId 8123000 already defined in " + + "[myserver/src/main/resources/transport/definitions/referable/existing_92.csv] contains id 8123000 already defined in " + "[myserver/src/main/resources/transport/definitions/referable/duplicate.csv]") } @@ -98,7 +98,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes def result = validateResourcesFails() then: assertValidateResourcesFailure(result, "Transport version definition file " + - "[myserver/src/main/resources/transport/definitions/referable/patched.csv] contains bwc definitionId [8100000] with a patch part of 0") + "[myserver/src/main/resources/transport/definitions/referable/patched.csv] contains bwc id [8100000] with a patch part of 0") } def "definitions have primary ids which cannot change"() { @@ -108,7 +108,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes def result = validateResourcesFails() then: assertValidateResourcesFailure(result, "Transport version definition file " + - "[myserver/src/main/resources/transport/definitions/referable/existing_92.csv] has modified primary definitionId from 8123000 to 8500000") + "[myserver/src/main/resources/transport/definitions/referable/existing_92.csv] has modified primary id from 8123000 to 8500000") } def "cannot change committed ids to a branch"() { @@ -118,7 +118,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes def result = validateResourcesFails() then: assertValidateResourcesFailure(result, "Transport version definition file " + - "[myserver/src/main/resources/transport/definitions/referable/existing_92.csv] modifies existing patch definitionId from 8012001 to 8012002") + "[myserver/src/main/resources/transport/definitions/referable/existing_92.csv] modifies existing patch id from 8012001 to 8012002") } def "upper bounds files must reference defined name"() { @@ -138,7 +138,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes def result = validateResourcesFails() then: assertValidateResourcesFailure(result, "Transport version upper bound file " + - "[myserver/src/main/resources/transport/upper_bounds/9.2.csv] has definitionId 8124000 which is not in definition " + + "[myserver/src/main/resources/transport/upper_bounds/9.2.csv] has id 8124000 which is not in definition " + "[myserver/src/main/resources/transport/definitions/referable/existing_92.csv]") } @@ -152,8 +152,8 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes def result = validateResourcesFails() then: assertValidateResourcesFailure(result, "Transport version upper bound file " + - "[myserver/src/main/resources/transport/upper_bounds/9.0.csv] has definitionId 8110001 from [seemingly_latest] with base 8110000 " + - "but another definitionId 8110002 from [actual_latest] is later for that base") + "[myserver/src/main/resources/transport/upper_bounds/9.0.csv] has id 8110001 from [seemingly_latest] with base 8110000 " + + "but another id 8110002 from [actual_latest] is later for that base") } def "upper bound files cannot change base id"() { @@ -165,7 +165,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes def result = validateResourcesFails() then: assertValidateResourcesFailure(result, "Transport version upper bound file " + - "[myserver/src/main/resources/transport/upper_bounds/9.1.csv] modifies base definitionId from 8012000 to 8013000") + "[myserver/src/main/resources/transport/upper_bounds/9.1.csv] modifies base id from 8012000 to 8013000") } def "ids must be dense"() { @@ -176,7 +176,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes when: def result = validateResourcesFails() then: - assertValidateResourcesFailure(result, "Transport version base definitionId 8013000 is missing patch ids between 8013000 and 8013002") + assertValidateResourcesFailure(result, "Transport version base id 8013000 is missing patch ids between 8013000 and 8013002") } def "primary id must not be patch version"() { @@ -186,7 +186,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes def result = validateResourcesFails() then: assertValidateResourcesFailure(result, "Transport version definition file " + - "[myserver/src/main/resources/transport/definitions/referable/patch.csv] has patch version 8015001 as primary definitionId") + "[myserver/src/main/resources/transport/definitions/referable/patch.csv] has patch version 8015001 as primary id") } def "unreferable directory is optional"() { @@ -238,7 +238,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes then: assertValidateResourcesFailure(result, "Transport version definition file " + "[myserver/src/main/resources/transport/definitions/referable/some_tv.csv] " + - "has the highest transport version definitionId [10000000] but is not present in any upper bounds files") + "has the highest transport version id [10000000] but is not present in any upper bounds files") } def "highest id in an unreferable definition should exist in an upper bounds file"() { @@ -250,6 +250,6 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes // TODO: this should be _unreferable_ in the error message, but will require some rework assertValidateResourcesFailure(result, "Transport version definition file " + "[myserver/src/main/resources/transport/definitions/referable/initial_10.0.0.csv] " + - "has the highest transport version definitionId [10000000] but is not present in any upper bounds files") + "has the highest transport version id [10000000] but is not present in any upper bounds files") } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionDefinition.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionDefinition.java index 4294f5e4e440a..65f4caeb95206 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionDefinition.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionDefinition.java @@ -25,7 +25,7 @@ public static TransportVersionDefinition fromString(Path file, String contents) try { ids.add(TransportVersionId.fromString(rawId)); } catch (NumberFormatException e) { - throw new IllegalStateException("Failed to parse definitionId " + rawId + " in " + file, e); + throw new IllegalStateException("Failed to parse id " + rawId + " in " + file, e); } } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java index f8d5dba73ee4b..cd5e61021a6e2 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java @@ -123,7 +123,7 @@ private Map> collectIdsByBase(Collection a.id().complete())); } @@ -157,18 +157,18 @@ private void validateNamedDefinition(TransportVersionDefinition definition, Set< if (ndx == 0) { if (getShouldValidatePrimaryIdNotPatch().get() && id.patch() != 0) { - throwDefinitionFailure(definition, "has patch version " + id.complete() + " as primary definitionId"); + throwDefinitionFailure(definition, "has patch version " + id.complete() + " as primary id"); } } else { if (id.patch() == 0) { - throwDefinitionFailure(definition, "contains bwc definitionId [" + id + "] with a patch part of 0"); + throwDefinitionFailure(definition, "contains bwc id [" + id + "] with a patch part of 0"); } } // check modifications of ids on same name, ie sharing same base TransportVersionId maybeModifiedId = existingIdsByBase.get(id.base()); if (maybeModifiedId != null && maybeModifiedId.complete() != id.complete()) { - throwDefinitionFailure(definition, "modifies existing patch definitionId from " + maybeModifiedId + " to " + id); + throwDefinitionFailure(definition, "modifies existing patch id from " + maybeModifiedId + " to " + id); } } } @@ -182,7 +182,7 @@ private void validateUnreferableDefinition(TransportVersionDefinition definition throwDefinitionFailure(definition, "does not contain any ids"); } if (definition.ids().size() > 1) { - throwDefinitionFailure(definition, " contains more than one definitionId"); + throwDefinitionFailure(definition, " contains more than one id"); } // note: no definitionName validation, anything that is a valid filename is ok, this allows eg initial_8.9.1 } @@ -193,7 +193,7 @@ private void validateIdenticalPrimaryId(TransportVersionDefinition definition, T int primaryId = definition.ids().get(0).complete(); int originalPrimaryId = originalDefinition.ids().get(0).complete(); if (primaryId != originalPrimaryId) { - throwDefinitionFailure(definition, "has modified primary definitionId from " + originalPrimaryId + " to " + primaryId); + throwDefinitionFailure(definition, "has modified primary id from " + originalPrimaryId + " to " + primaryId); } } @@ -213,7 +213,7 @@ private void validateUpperBound( Path relativePath = getResources().get().getReferableDefinitionRepositoryPath(upperBoundDefinition); throwUpperBoundFailure( upperBound, - "has definitionId " + upperBound.definitionId() + " which is not in definition [" + relativePath + "]" + "has id " + upperBound.definitionId() + " which is not in definition [" + relativePath + "]" ); } @@ -222,13 +222,13 @@ private void validateUpperBound( if (lastId.id().complete() != upperBound.definitionId().complete()) { throwUpperBoundFailure( upperBound, - "has definitionId " + "has id " + upperBound.definitionId() + " from [" + upperBound.definitionName() + "] with base " + upperBound.definitionId().base() - + " but another definitionId " + + " but another id " + lastId.id().complete() + " from [" + lastId.definition().name() @@ -241,7 +241,7 @@ private void validateUpperBound( if (upperBound.definitionId().patch() != 0 && upperBound.definitionId().base() != existingUpperBound.definitionId().base()) { throwUpperBoundFailure( upperBound, - "modifies base definitionId from " + "modifies base id from " + existingUpperBound.definitionId().base() + " to " + upperBound.definitionId().base() @@ -260,13 +260,13 @@ private void validateBase(int base, List ids) { Path existingDefinitionPath = getResources().get().getReferableDefinitionRepositoryPath(previous.definition); throwDefinitionFailure( current.definition(), - "contains definitionId " + current.id + " already defined in [" + existingDefinitionPath + "]" + "contains id " + current.id + " already defined in [" + existingDefinitionPath + "]" ); } if (getShouldValidateDensity().get() && previous.id().complete() - 1 != current.id().complete()) { throw new IllegalStateException( - "Transport version base definitionId " + "Transport version base id " + base + " is missing patch ids between " + current.id() @@ -282,8 +282,8 @@ private void validateLargestIdIsUsed( Map upperBounds, Map allDefinitions ) { - // first definitionId is always the highest within a definition, and validated earlier - // note we use min instead of max because the definitionId comparator is in descending order + // first id is always the highest within a definition, and validated earlier + // note we use min instead of max because the id comparator is in descending order var highestDefinition = allDefinitions.values().stream().min(Comparator.comparing(d -> d.ids().get(0))).get(); var highestId = highestDefinition.ids().get(0); @@ -295,7 +295,7 @@ private void validateLargestIdIsUsed( throwDefinitionFailure( highestDefinition, - "has the highest transport version definitionId [" + highestId + "] but is not present in any upper bounds files" + "has the highest transport version id [" + highestId + "] but is not present in any upper bounds files" ); } From 0a1898cca3d2ecb66c33d80e7a9cadbb079b4a2b Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 5 Sep 2025 00:21:35 +0000 Subject: [PATCH 66/68] [CI] Auto commit changes from spotless --- .../ValidateTransportVersionResourcesTask.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java index cd5e61021a6e2..a1cf87a777ad0 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java @@ -241,10 +241,7 @@ private void validateUpperBound( if (upperBound.definitionId().patch() != 0 && upperBound.definitionId().base() != existingUpperBound.definitionId().base()) { throwUpperBoundFailure( upperBound, - "modifies base id from " - + existingUpperBound.definitionId().base() - + " to " - + upperBound.definitionId().base() + "modifies base id from " + existingUpperBound.definitionId().base() + " to " + upperBound.definitionId().base() ); } } @@ -266,12 +263,7 @@ private void validateBase(int base, List ids) { if (getShouldValidateDensity().get() && previous.id().complete() - 1 != current.id().complete()) { throw new IllegalStateException( - "Transport version base id " - + base - + " is missing patch ids between " - + current.id() - + " and " - + previous.id() + "Transport version base id " + base + " is missing patch ids between " + current.id() + " and " + previous.id() ); } previous = current; From 68a4f71ec3a1c0a8fe3c48eb937b3cee9e631647 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 4 Sep 2025 21:10:26 -0700 Subject: [PATCH 67/68] fix refactor --- .../internal/transport/AbstractTransportVersionFuncTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy index a9b7f2c2398b6..742fb80838fdc 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy @@ -148,7 +148,7 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest { void setupLocalGitRepo() { execute("git init") execute('git config user.email "build-tool@elastic.co"') - execute('git config user.definitionName "Build tool"') + execute('git config user.name "Build tool"') execute("git add .") execute('git commit -m "Initial"') } From 012999a5542d42146d2c0ffe9f3b489224be2a36 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 4 Sep 2025 21:17:59 -0700 Subject: [PATCH 68/68] fix more refactor problems --- .../AbstractTransportVersionFuncTest.groovy | 2 +- .../TransportVersionValidationFuncTest.groovy | 6 +++--- .../GenerateTransportVersionDefinitionTask.java | 16 ++++++++-------- .../ValidateTransportVersionResourcesTask.java | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy index 742fb80838fdc..edfca7e6918b5 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy @@ -122,7 +122,7 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest { transportVersionUpperBound("9.0", "initial_9_0_0", "8000000") // a mock version of TransportVersion, just here so we can compile Dummy.java et al javaSource("myserver", "org.elasticsearch", "TransportVersion", "", """ - public static TransportVersion fromName(String definitionName) { + public static TransportVersion fromName(String name) { return null; } """) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy index 989954a5407e8..36be4c4d94bff 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy @@ -53,7 +53,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes def result = validateResourcesFails() then: assertValidateResourcesFailure(result, "Transport version definition file " + - "[myserver/src/main/resources/transport/definitions/referable/${name}.csv] does not have a valid definitionName, " + + "[myserver/src/main/resources/transport/definitions/referable/${name}.csv] does not have a valid name, " + "must be lowercase alphanumeric and underscore") where: @@ -128,7 +128,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes def result = validateResourcesFails() then: assertValidateResourcesFailure(result, "Transport version upper bound file " + - "[myserver/src/main/resources/transport/upper_bounds/9.2.csv] contains transport version definitionName [dne] which is not defined") + "[myserver/src/main/resources/transport/upper_bounds/9.2.csv] contains transport version name [dne] which is not defined") } def "upper bound files id must exist in definition"() { @@ -217,7 +217,7 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes then: assertValidateResourcesFailure(result, "Transport version definition file " + "[myserver/src/main/resources/transport/definitions/referable/existing_92.csv] " + - "has same definitionName as unreferable definition " + + "has same name as unreferable definition " + "[myserver/src/main/resources/transport/definitions/unreferable/existing_92.csv]") } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index a4918f79bc6ae..8c45823ea6de0 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -32,7 +32,7 @@ * This task generates transport version definition files. These files * are runtime resources that TransportVersion loads statically. * They contain a comma separated list of integer ids. Each file is named the same - * as the transport version definitionName itself (with the .csv suffix). + * as the transport version name itself (with the .csv suffix). * * Additionally, when definition files are added or updated, the upper bounds files * for each relevant branch's upper bound file are also updated. @@ -140,21 +140,21 @@ private String getTargetDefinitionName( List changedDefinitions ) { if (getDefinitionName().isPresent()) { - // an explicit definitionName was passed in, so use it + // an explicit name was passed in, so use it return getDefinitionName().get(); } // First check for unreferenced names. We only care about the first one. If there is more than one // validation will fail later and the developer will have to remove one. When that happens, generation - // will re-run and we will fixup the state to use whatever new definitionName remains. + // will re-run and we will fixup the state to use whatever new name remains. for (String referencedName : referencedNames) { if (resources.referableDefinitionExists(referencedName) == false) { return referencedName; } } - // Since we didn't find any missing names, we use the first changed definitionName. If there is more than - // one changed definitionName, validation will fail later, just as above. + // Since we didn't find any missing names, we use the first changed name. If there is more than + // one changed name, validation will fail later, just as above. if (changedDefinitions.isEmpty()) { return ""; } else { @@ -162,7 +162,7 @@ private String getTargetDefinitionName( if (referencedNames.contains(changedDefinitionName)) { return changedDefinitionName; } else { - return ""; // the changed definitionName is unreferenced, so go into "reset mode" + return ""; // the changed name is unreferenced, so go into "reset mode" } } } @@ -217,11 +217,11 @@ private TransportVersionId maybeGetExistingId( String name ) { if (existingDefinition == null) { - // the definitionName doesn't yet exist, so there is no id to return + // the name doesn't yet exist, so there is no id to return return null; } if (upperBound.definitionName().equals(name)) { - // the definitionName exists and this upper bound already points at it + // the name exists and this upper bound already points at it return upperBound.definitionId(); } if (upperBound.name().equals(getCurrentUpperBoundName().get())) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java index a1cf87a777ad0..9080de1fd79a5 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java @@ -105,7 +105,7 @@ private Map collectAllDefinitions( TransportVersionDefinition existing = allDefinitions.put(entry.getKey(), entry.getValue()); if (existing != null) { Path unreferablePath = getResources().get().getUnreferableDefinitionRepositoryPath(entry.getValue()); - throwDefinitionFailure(existing, "has same definitionName as unreferable definition [" + unreferablePath + "]"); + throwDefinitionFailure(existing, "has same name as unreferable definition [" + unreferablePath + "]"); } } return allDefinitions; @@ -144,7 +144,7 @@ private void validateNamedDefinition(TransportVersionDefinition definition, Set< throwDefinitionFailure(definition, "is not referenced"); } if (NAME_FORMAT.matcher(definition.name()).matches() == false) { - throwDefinitionFailure(definition, "does not have a valid definitionName, must be lowercase alphanumeric and underscore"); + throwDefinitionFailure(definition, "does not have a valid name, must be lowercase alphanumeric and underscore"); } if (definition.ids().isEmpty()) { throwDefinitionFailure(definition, "does not contain any ids"); @@ -184,7 +184,7 @@ private void validateUnreferableDefinition(TransportVersionDefinition definition if (definition.ids().size() > 1) { throwDefinitionFailure(definition, " contains more than one id"); } - // note: no definitionName validation, anything that is a valid filename is ok, this allows eg initial_8.9.1 + // note: no name validation, anything that is a valid filename is ok, this allows eg initial_8.9.1 } private void validateIdenticalPrimaryId(TransportVersionDefinition definition, TransportVersionDefinition originalDefinition) { @@ -206,7 +206,7 @@ private void validateUpperBound( if (upperBoundDefinition == null) { throwUpperBoundFailure( upperBound, - "contains transport version definitionName [" + upperBound.definitionName() + "] which is not defined" + "contains transport version name [" + upperBound.definitionName() + "] which is not defined" ); } if (upperBoundDefinition.ids().contains(upperBound.definitionId()) == false) {