Backport several transport version fixes

This commit backports several changes to the new transport version system

relates #133034
relates #133185
relates #133186
relates #133189

Author: rjernst

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy

@@ -41,11 +41,11 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
         javaResource("myserver", "transport/definitions/unreferenced/" + name + ".csv", id)
     }
 
-    def definedAndUsedTransportVersion(String name, String ids) {
-        return definedAndUsedTransportVersion(name, ids, "Test${name.capitalize()}")
+    def namedAndReferencedTransportVersion(String name, String ids) {
+        return namedAndReferencedTransportVersion(name, ids, "Test${name.capitalize()}")
     }
 
-    def definedAndUsedTransportVersion(String name, String ids, String classname) {
+    def namedAndReferencedTransportVersion(String name, String ids, String classname) {
         javaSource("myserver", "org.elasticsearch", classname, "", """
             static final TransportVersion usage = TransportVersion.fromName("${name}");
         """)
@@ -60,17 +60,17 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
         return gradleRunner(":${project}:validateTransportVersionReferences").buildAndFail()
     }
 
-    def validateDefinitionsFails() {
-        return gradleRunner(":myserver:validateTransportVersionDefinitions").buildAndFail()
+    def validateResourcesFails() {
+        return gradleRunner(":myserver:validateTransportVersionResources").buildAndFail()
     }
 
-    def assertReferencesFailure(BuildResult result, String project, String expectedOutput) {
+    def assertValidateReferencesFailure(BuildResult result, String project, String expectedOutput) {
         result.task(":${project}:validateTransportVersionReferences").outcome == TaskOutcome.FAILED
         assertOutputContains(result.output, expectedOutput)
     }
 
-    def assertDefinitionsFailure(BuildResult result, String expectedOutput) {
-        result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.FAILED
+    def assertValidateResourcesFailure(BuildResult result, String expectedOutput) {
+        result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.FAILED
         assertOutputContains(result.output, expectedOutput)
     }
 
@@ -81,9 +81,6 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
             include ':myserver'
             include ':myplugin'
         """
-        file("gradle.properties") << """
-            org.elasticsearch.transport.definitionsProject=:myserver
-        """
 
         file("myserver/build.gradle") << """
             apply plugin: 'java-library'

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy

@@ -16,9 +16,9 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes
 
     def "test setup works"() {
         when:
-        def result = gradleRunner("validateTransportVersionDefinitions", "validateTransportVersionReferences").build()
+        def result = gradleRunner("validateTransportVersionResources", "validateTransportVersionReferences").build()
         then:
-        result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS
+        result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
         result.task(":myserver:validateTransportVersionReferences").outcome == TaskOutcome.SUCCESS
         result.task(":myplugin:validateTransportVersionReferences").outcome == TaskOutcome.SUCCESS
     }
@@ -32,27 +32,27 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes
         when:
         def result = validateReferencesFails("myplugin")
         then:
-        assertReferencesFailure(result, "myplugin", "TransportVersion.fromName(\"dne\") was used at " +
+        assertValidateReferencesFailure(result, "myplugin", "TransportVersion.fromName(\"dne\") was used at " +
                 "org.elasticsearch.plugin.MyPlugin line 6, but lacks a transport version definition.")
     }
 
     def "references must be defined"() {
         given:
         namedTransportVersion("not_used", "1000000")
         when:
-        def result = validateDefinitionsFails()
+        def result = validateResourcesFails()
         then:
-        assertDefinitionsFailure(result, "Transport version definition file " +
+        assertValidateResourcesFailure(result, "Transport version definition file " +
             "[myserver/src/main/resources/transport/definitions/named/not_used.csv] is not referenced")
     }
 
     def "names must be lowercase alphanum or underscore"() {
         given:
-        definedAndUsedTransportVersion("${name}", "8100000", "TestNames")
+        namedAndReferencedTransportVersion("${name}", "8100000", "TestNames")
         when:
-        def result = validateDefinitionsFails()
+        def result = validateResourcesFails()
         then:
-        assertDefinitionsFailure(result, "Transport version definition file " +
+        assertValidateResourcesFailure(result, "Transport version definition file " +
             "[myserver/src/main/resources/transport/definitions/named/${name}.csv] does not have a valid name, " +
             "must be lowercase alphanumeric and underscore")
 
@@ -62,130 +62,130 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes
 
     def "definitions contain at least one id"() {
         given:
-        definedAndUsedTransportVersion("empty", "")
+        namedAndReferencedTransportVersion("empty", "")
         when:
-        def result = validateDefinitionsFails()
+        def result = validateResourcesFails()
         then:
-        assertDefinitionsFailure(result, "Transport version definition file " +
+        assertValidateResourcesFailure(result, "Transport version definition file " +
             "[myserver/src/main/resources/transport/definitions/named/empty.csv] does not contain any ids")
     }
 
     def "definitions have ids in descending order"() {
         given:
-        definedAndUsedTransportVersion("out_of_order", "8100000,8200000")
+        namedAndReferencedTransportVersion("out_of_order", "8100000,8200000")
         when:
-        def result = validateDefinitionsFails()
+        def result = validateResourcesFails()
         then:
-        assertDefinitionsFailure(result, "Transport version definition file " +
+        assertValidateResourcesFailure(result, "Transport version definition file " +
             "[myserver/src/main/resources/transport/definitions/named/out_of_order.csv] does not have ordered ids")
     }
 
     def "definition ids are unique"() {
         given:
-        definedAndUsedTransportVersion("duplicate", "8123000")
+        namedAndReferencedTransportVersion("duplicate", "8123000")
         when:
-        def result = validateDefinitionsFails()
+        def result = validateResourcesFails()
         then:
-        assertDefinitionsFailure(result, "Transport version definition file " +
+        assertValidateResourcesFailure(result, "Transport version definition file " +
             "[myserver/src/main/resources/transport/definitions/named/existing_92.csv] contains id 8123000 already defined in " +
             "[myserver/src/main/resources/transport/definitions/named/duplicate.csv]")
     }
 
     def "definitions have bwc ids with non-zero patch part"() {
         given:
-        definedAndUsedTransportVersion("patched", "8200000,8100000")
+        namedAndReferencedTransportVersion("patched", "8200000,8100000")
         when:
-        def result = validateDefinitionsFails()
+        def result = validateResourcesFails()
         then:
-        assertDefinitionsFailure(result, "Transport version definition file " +
+        assertValidateResourcesFailure(result, "Transport version definition file " +
             "[myserver/src/main/resources/transport/definitions/named/patched.csv] contains bwc id [8100000] with a patch part of 0")
     }
 
     def "definitions have primary ids which cannot change"() {
         given:
         namedTransportVersion("existing_92", "8500000")
         when:
-        def result = validateDefinitionsFails()
+        def result = validateResourcesFails()
         then:
-        assertDefinitionsFailure(result, "Transport version definition file " +
+        assertValidateResourcesFailure(result, "Transport version definition file " +
             "[myserver/src/main/resources/transport/definitions/named/existing_92.csv] has modified primary id from 8123000 to 8500000")
     }
 
     def "cannot change committed ids to a branch"() {
         given:
         namedTransportVersion("existing_92", "8123000,8012002")
         when:
-        def result = validateDefinitionsFails()
+        def result = validateResourcesFails()
         then:
-        assertDefinitionsFailure(result, "Transport version definition file " +
+        assertValidateResourcesFailure(result, "Transport version definition file " +
             "[myserver/src/main/resources/transport/definitions/named/existing_92.csv] modifies existing patch id from 8012001 to 8012002")
     }
 
     def "latest files must reference defined name"() {
         given:
         latestTransportVersion("9.2", "dne", "8123000")
         when:
-        def result = validateDefinitionsFails()
+        def result = validateResourcesFails()
         then:
-        assertDefinitionsFailure(result, "Latest transport version file " +
+        assertValidateResourcesFailure(result, "Latest transport version file " +
             "[myserver/src/main/resources/transport/latest/9.2.csv] contains transport version name [dne] which is not defined")
     }
 
     def "latest files id must exist in definition"() {
         given:
         latestTransportVersion("9.2", "existing_92", "8124000")
         when:
-        def result = validateDefinitionsFails()
+        def result = validateResourcesFails()
         then:
-        assertDefinitionsFailure(result, "Latest transport version file " +
+        assertValidateResourcesFailure(result, "Latest transport version file " +
             "[myserver/src/main/resources/transport/latest/9.2.csv] has id 8124000 which is not in definition " +
             "[myserver/src/main/resources/transport/definitions/named/existing_92.csv]")
     }
 
     def "latest files have latest id within base"() {
         given:
         latestTransportVersion("9.0", "seemingly_latest", "8110001")
-        definedAndUsedTransportVersion("original", "8110000")
-        definedAndUsedTransportVersion("seemingly_latest", "8111000,8110001")
-        definedAndUsedTransportVersion("actual_latest", "8112000,8110002")
+        namedAndReferencedTransportVersion("original", "8110000")
+        namedAndReferencedTransportVersion("seemingly_latest", "8111000,8110001")
+        namedAndReferencedTransportVersion("actual_latest", "8112000,8110002")
         when:
-        def result = validateDefinitionsFails()
+        def result = validateResourcesFails()
         then:
-        assertDefinitionsFailure(result, "Latest transport version file " +
+        assertValidateResourcesFailure(result, "Latest transport version file " +
             "[myserver/src/main/resources/transport/latest/9.0.csv] has id 8110001 from [seemingly_latest] with base 8110000 " +
             "but another id 8110002 from [actual_latest] is later for that base")
     }
 
     def "latest files cannot change base id"() {
         given:
-        definedAndUsedTransportVersion("original", "8013000")
-        definedAndUsedTransportVersion("patch", "8015000,8013001")
+        namedAndReferencedTransportVersion("original", "8013000")
+        namedAndReferencedTransportVersion("patch", "8015000,8013001")
         latestTransportVersion("9.1", "patch", "8013001")
         when:
-        def result = validateDefinitionsFails()
+        def result = validateResourcesFails()
         then:
-        assertDefinitionsFailure(result, "Latest transport version file " +
+        assertValidateResourcesFailure(result, "Latest transport version file " +
             "[myserver/src/main/resources/transport/latest/9.1.csv] modifies base id from 8012000 to 8013000")
     }
 
     def "ids must be dense"() {
         given:
-        definedAndUsedTransportVersion("original", "8013000")
-        definedAndUsedTransportVersion("patch1", "8015000,8013002")
+        namedAndReferencedTransportVersion("original", "8013000")
+        namedAndReferencedTransportVersion("patch1", "8015000,8013002")
         latestTransportVersion("9.0", "patch1", "8013002")
         when:
-        def result = validateDefinitionsFails()
+        def result = validateResourcesFails()
         then:
-        assertDefinitionsFailure(result, "Transport version base id 8013000 is missing patch ids between 8013000 and 8013002")
+        assertValidateResourcesFailure(result, "Transport version base id 8013000 is missing patch ids between 8013000 and 8013002")
     }
 
     def "primary id must not be patch version"() {
         given:
-        definedAndUsedTransportVersion("patch", "8015001")
+        namedAndReferencedTransportVersion("patch", "8015001")
         when:
-        def result = validateDefinitionsFails()
+        def result = validateResourcesFails()
         then:
-        assertDefinitionsFailure(result, "Transport version definition file " +
+        assertValidateResourcesFailure(result, "Transport version definition file " +
             "[myserver/src/main/resources/transport/definitions/named/patch.csv] has patch version 8015001 as primary id")
     }
 
@@ -194,8 +194,39 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes
         file("myserver/src/main/resources/transport/unreferenced/initial_9_0_0.csv").delete()
         file("myserver/src/main/resources/transport/unreferenced").deleteDir()
         when:
-        def result = gradleRunner(":myserver:validateTransportVersionDefinitions").build()
+        def result = gradleRunner(":myserver:validateTransportVersionResources").build()
         then:
-        result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS
+        result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
+    }
+
+    def "latest can refer to an unreferenced definition"() {
+        given:
+        unreferencedTransportVersion("initial_10.0.0", "10000000")
+        latestTransportVersion("10.0", "initial_10.0.0", "10000000")
+        when:
+        def result = gradleRunner(":myserver:validateTransportVersionResources").build()
+        then:
+        result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
+    }
+
+    def "named and unreferenced definitions cannot have the same name"() {
+        given:
+        unreferencedTransportVersion("existing_92", "10000000")
+        when:
+        def result = validateResourcesFails()
+        then:
+        assertValidateResourcesFailure(result, "Transport version definition file " +
+                "[myserver/src/main/resources/transport/definitions/named/existing_92.csv] " +
+                "has same name as unreferenced definition " +
+                "[myserver/src/main/resources/transport/definitions/unreferenced/existing_92.csv]")
+    }
+
+    def "unreferenced definitions can have primary ids that are patches"() {
+        given:
+        unreferencedTransportVersion("initial_10.0.1", "10000001")
+        when:
+        def result = gradleRunner(":myserver:validateTransportVersionResources").build()
+        then:
+        result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
     }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java

@@ -10,8 +10,9 @@
 package org.elasticsearch.gradle.internal.transport;
 
 import org.gradle.api.DefaultTask;
-import org.gradle.api.file.DirectoryProperty;
 import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.provider.Property;
+import org.gradle.api.services.ServiceReference;
 import org.gradle.api.tasks.InputDirectory;
 import org.gradle.api.tasks.Optional;
 import org.gradle.api.tasks.OutputFile;
@@ -28,24 +29,25 @@
 import java.nio.file.attribute.BasicFileAttributes;
 
 public abstract class GenerateTransportVersionManifestTask extends DefaultTask {
+
+    @ServiceReference("transportVersionResources")
+    abstract Property<TransportVersionResourcesService> getTransportResources();
+
     @InputDirectory
     @Optional
     @PathSensitive(PathSensitivity.RELATIVE)
-    public abstract DirectoryProperty getDefinitionsDirectory();
+    public Path getDefinitionsDirectory() {
+        return getTransportResources().get().getDefinitionsDir();
+    }
 
     @OutputFile
     public abstract RegularFileProperty getManifestFile();
 
     @TaskAction
     public void generateTransportVersionManifest() throws IOException {
-        Path definitionsDir = getDefinitionsDirectory().get().getAsFile().toPath();
-        Path manifestFile = getManifestFile().get().getAsFile().toPath();
-        if (getDefinitionsDirectory().isPresent() == false) {
-            // no definitions to capture, remove this leniency once all branches have at least one version
-            Files.writeString(manifestFile, "", StandardCharsets.UTF_8);
-            return;
-        }
 
+        Path definitionsDir = getDefinitionsDirectory();
+        Path manifestFile = getManifestFile().get().getAsFile().toPath();
         try (var writer = Files.newBufferedWriter(manifestFile)) {
             Files.walkFileTree(definitionsDir, new SimpleFileVisitor<>() {
                 @Override

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionId.java

@@ -22,7 +22,8 @@ static TransportVersionId fromString(String s) {
 
     @Override
     public int compareTo(TransportVersionId o) {
-        return Integer.compare(complete, o.complete);
+        // note: this is descending order so the arguments are reversed
+        return Integer.compare(o.complete, complete);
     }
 
     @Override

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionReference.java

@@ -12,12 +12,15 @@
 import org.gradle.api.attributes.Attribute;
 import org.gradle.api.attributes.AttributeContainer;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE;
 
@@ -43,6 +46,14 @@ static void addArtifactAttribute(AttributeContainer attributes) {
         attributes.attribute(REFERENCES_ATTRIBUTE, true);
     }
 
+    static Set<String> collectNames(Iterable<File> referencesFiles) throws IOException {
+        Set<String> names = new HashSet<>();
+        for (var referencesFile : referencesFiles) {
+            listFromFile(referencesFile.toPath()).stream().map(TransportVersionReference::name).forEach(names::add);
+        }
+        return names;
+    }
+
     @Override
     public String toString() {
         return name + "," + location;