Change upstreamRef override to baseRef override (elastic#135807)

When calculating the base git reference to read transport version
resources from the upstream main ref must be known. Before we calculated
the merge base this was overridden with a gradle property by tests and
release tooling. However, in the case of release tooling we actually
want to override the base ref directly so that it can be run on non-main
branches.

This commit renames the upstreamRef gradle property to baseRef. It also
renames the resources service methods to make clearer what they are
reading: from the git (merge) base.

Author: rjernst

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy

@@ -115,9 +115,6 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
             include ':myserver'
             include ':myplugin'
         """
-        propertiesFile << """
-            org.elasticsearch.transport.upstreamRef=main
-        """
         versionPropertiesFile.text = versionPropertiesFile.text.replace("9.1.0", "9.2.0")
 
         file("myserver/build.gradle") << """

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateInitialTransportVersionTask.java

@@ -37,31 +37,31 @@ public void run() throws IOException {
         Version releaseVersion = Version.fromString(getReleaseVersion().get());
         String upperBoundName = getUpperBoundName(releaseVersion);
         TransportVersionResourcesService resources = getResourceService().get();
-        TransportVersionUpperBound upstreamUpperBound = resources.getUpperBoundFromUpstream(upperBoundName);
+        TransportVersionUpperBound baseUpperBound = resources.getUpperBoundFromGitBase(upperBoundName);
         String initialDefinitionName = "initial_" + releaseVersion;
-        TransportVersionDefinition existingDefinition = resources.getUnreferableDefinitionFromUpstream(initialDefinitionName);
+        TransportVersionDefinition existingDefinition = resources.getUnreferableDefinitionFromGitBase(initialDefinitionName);
 
-        if (existingDefinition != null) {
-            // this initial version has already been created upstream
-            return;
-        }
+        // This task runs on main and release branches. In release branches we will generate the exact same
+        // upper bound result because we always look at the base branch (ie upstream/main).
+        if (existingDefinition == null) {
+            if (baseUpperBound == null) {
+                throw new RuntimeException("Missing upper bound " + upperBoundName + " for release version " + releaseVersion);
+            }
 
-        if (upstreamUpperBound == null) {
-            throw new RuntimeException("Missing upper bound " + upperBoundName + " for release version " + releaseVersion);
-        }
-        // minors increment by 1000 to create a unique base, patches increment by 1 as other patches do
-        int increment = releaseVersion.getRevision() == 0 ? 1000 : 1;
-        var id = TransportVersionId.fromInt(upstreamUpperBound.definitionId().complete() + increment);
-        var definition = new TransportVersionDefinition(initialDefinitionName, List.of(id), false);
-        resources.writeDefinition(definition);
-        var newUpperBound = new TransportVersionUpperBound(upperBoundName, initialDefinitionName, id);
-        resources.writeUpperBound(newUpperBound, false);
+            // minors increment by 1000 to create a unique base, patches increment by 1 as other patches do
+            int increment = releaseVersion.getRevision() == 0 ? 1000 : 1;
+            var id = TransportVersionId.fromInt(baseUpperBound.definitionId().complete() + increment);
+            var definition = new TransportVersionDefinition(initialDefinitionName, List.of(id), false);
+            resources.writeDefinition(definition);
+            var newUpperBound = new TransportVersionUpperBound(upperBoundName, initialDefinitionName, id);
+            resources.writeUpperBound(newUpperBound, false);
 
-        if (releaseVersion.getRevision() == 0) {
-            Version currentVersion = getCurrentVersion().get();
-            String currentUpperBoundName = getUpperBoundName(currentVersion);
-            var currentUpperBound = new TransportVersionUpperBound(currentUpperBoundName, initialDefinitionName, id);
-            resources.writeUpperBound(currentUpperBound, false);
+            if (releaseVersion.getRevision() == 0) {
+                Version currentVersion = getCurrentVersion().get();
+                String currentUpperBoundName = getUpperBoundName(currentVersion);
+                var currentUpperBound = new TransportVersionUpperBound(currentUpperBoundName, initialDefinitionName, id);
+                resources.writeUpperBound(currentUpperBound, false);
+            }
         }
     }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java

@@ -109,7 +109,7 @@ public void run() throws IOException {
             resetAllUpperBounds(resources, idsByBase);
         } else {
             getLogger().lifecycle("Generating transport version name: " + targetDefinitionName);
-            List<TransportVersionUpperBound> upstreamUpperBounds = resources.getUpperBoundsFromUpstream();
+            List<TransportVersionUpperBound> upstreamUpperBounds = resources.getUpperBoundsFromGitBase();
             Set<String> targetUpperBoundNames = getTargetUpperBoundNames(resources, upstreamUpperBounds, targetDefinitionName);
 
             List<TransportVersionId> ids = updateUpperBounds(
@@ -143,7 +143,7 @@ private List<TransportVersionId> updateUpperBounds(
         List<TransportVersionId> ids = new ArrayList<>();
         boolean stageInGit = getResolveConflict().getOrElse(false);
 
-        TransportVersionDefinition existingDefinition = resources.getReferableDefinitionFromUpstream(definitionName);
+        TransportVersionDefinition existingDefinition = resources.getReferableDefinitionFromGitBase(definitionName);
         for (TransportVersionUpperBound existingUpperBound : existingUpperBounds) {
             String upperBoundName = existingUpperBound.name();
 
@@ -263,7 +263,7 @@ private Set<String> getUpperBoundNamesFromDefinition(
     private void resetAllUpperBounds(TransportVersionResourcesService resources, Map<Integer, List<IdAndDefinition>> idsByBase)
         throws IOException {
         for (String upperBoundName : resources.getChangedUpperBoundNames()) {
-            TransportVersionUpperBound upstreamUpperBound = resources.getUpperBoundFromUpstream(upperBoundName);
+            TransportVersionUpperBound upstreamUpperBound = resources.getUpperBoundFromGitBase(upperBoundName);
             resetUpperBound(resources, upstreamUpperBound, idsByBase, null);
         }
     }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java

@@ -50,9 +50,9 @@ public void apply(Project project) {
                 Directory transportResources = project.getLayout().getProjectDirectory().dir("src/main/resources/" + resourceRoot);
                 spec.getParameters().getTransportResourcesDirectory().set(transportResources);
                 spec.getParameters().getRootDirectory().set(project.getLayout().getSettingsDirectory().getAsFile());
-                Provider<String> upstreamRef = project.getProviders().gradleProperty("org.elasticsearch.transport.upstreamRef");
+                Provider<String> upstreamRef = project.getProviders().gradleProperty("org.elasticsearch.transport.baseRef");
                 if (upstreamRef.isPresent()) {
-                    spec.getParameters().getUpstreamRefOverride().set(upstreamRef.get());
+                    spec.getParameters().getBaseRefOverride().set(upstreamRef.get());
                 }
             });
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java

@@ -66,7 +66,7 @@ public interface Parameters extends BuildServiceParameters {
         DirectoryProperty getRootDirectory();
 
         @Optional
-        Property<String> getUpstreamRefOverride();
+        Property<String> getBaseRefOverride();
     }
 
     record IdAndDefinition(TransportVersionId id, TransportVersionDefinition definition) {}
@@ -81,7 +81,6 @@ record IdAndDefinition(TransportVersionId id, TransportVersionDefinition definit
 
     private final Path transportResourcesDir;
     private final Path rootDir;
-    private final String upstreamRefOverride;
     private final AtomicReference<String> baseRefName = new AtomicReference<>();
     private final AtomicReference<Set<String>> upstreamResources = new AtomicReference<>(null);
     private final AtomicReference<Set<String>> changedResources = new AtomicReference<>(null);
@@ -90,7 +89,9 @@ record IdAndDefinition(TransportVersionId id, TransportVersionDefinition definit
     public TransportVersionResourcesService(Parameters params) {
         this.transportResourcesDir = params.getTransportResourcesDirectory().get().getAsFile().toPath();
         this.rootDir = params.getRootDirectory().get().getAsFile().toPath();
-        upstreamRefOverride = params.getUpstreamRefOverride().getOrNull();
+        if (params.getBaseRefOverride().isPresent()) {
+            this.baseRefName.set(params.getBaseRefOverride().get());
+        }
     }
 
     /**
@@ -126,8 +127,8 @@ TransportVersionDefinition getReferableDefinition(String name) throws IOExceptio
         return TransportVersionDefinition.fromString(resourcePath, Files.readString(resourcePath, StandardCharsets.UTF_8), true);
     }
 
-    /** Get a referable definition from upstream if it exists there, or null otherwise */
-    TransportVersionDefinition getReferableDefinitionFromUpstream(String name) {
+    /** Get a referable definition from the merge base ref in git if it exists there, or null otherwise */
+    TransportVersionDefinition getReferableDefinitionFromGitBase(String name) {
         Path resourcePath = getDefinitionRelativePath(name, true);
         return getUpstreamFile(resourcePath, (path, contents) -> TransportVersionDefinition.fromString(path, contents, true));
     }
@@ -174,8 +175,8 @@ Map<String, TransportVersionDefinition> getUnreferableDefinitions() throws IOExc
         return readDefinitions(transportResourcesDir.resolve(UNREFERABLE_DIR), false);
     }
 
-    /** Get a referable definition from upstream if it exists there, or null otherwise */
-    TransportVersionDefinition getUnreferableDefinitionFromUpstream(String name) {
+    /** Get a referable definition from the merge base ref in git if it exists there, or null otherwise */
+    TransportVersionDefinition getUnreferableDefinitionFromGitBase(String name) {
         Path resourcePath = getDefinitionRelativePath(name, false);
         return getUpstreamFile(resourcePath, (path, contents) -> TransportVersionDefinition.fromString(path, contents, false));
     }
@@ -214,14 +215,14 @@ Map<String, TransportVersionUpperBound> getUpperBounds() throws IOException {
         return upperBounds;
     }
 
-    /** Retrieve an upper bound from upstream by name  */
-    TransportVersionUpperBound getUpperBoundFromUpstream(String name) {
+    /** Retrieve an upper bound from the merge base ref in git by name  */
+    TransportVersionUpperBound getUpperBoundFromGitBase(String name) {
         Path resourcePath = getUpperBoundRelativePath(name);
         return getUpstreamFile(resourcePath, TransportVersionUpperBound::fromString);
     }
 
-    /** Retrieve all upper bounds that exist in upstream */
-    List<TransportVersionUpperBound> getUpperBoundsFromUpstream() throws IOException {
+    /** Retrieve all upper bounds that exist in the merge base ref in git */
+    List<TransportVersionUpperBound> getUpperBoundsFromGitBase() throws IOException {
         List<TransportVersionUpperBound> upperBounds = new ArrayList<>();
         for (String upstreamPathString : getUpstreamResources()) {
             Path upstreamPath = Path.of(upstreamPathString);
@@ -278,15 +279,10 @@ private String getBaseRefName() {
     }
 
     private String findUpstreamRef() {
-        if (upstreamRefOverride != null) {
-            return upstreamRefOverride;
-        }
-
         String remotesOutput = gitCommand("remote").strip();
         if (remotesOutput.isEmpty()) {
-            throw new RuntimeException(
-                "No remotes found. If this is a test set gradle property " + "org.elasticsearch.transport.upstreamRef"
-            );
+            logger.warn("No remotes found. Using 'main' branch as upstream ref for transport version resources");
+            return "main";
         }
         List<String> remoteNames = List.of(remotesOutput.split("\n"));
         if (remoteNames.contains(UPSTREAM_REMOTE_NAME) == false) {
@@ -302,7 +298,8 @@ private String findUpstreamRef() {
             if (upstreamUrl != null) {
                 gitCommand("remote", "add", UPSTREAM_REMOTE_NAME, upstreamUrl);
             } else {
-                throw new RuntimeException("No elastic github remotes found to copy");
+                logger.warn("No elastic github remotes found to copy. Using 'main' branch as upstream ref for transport version resources");
+                return "main";
             }
         }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java

@@ -153,7 +153,7 @@ private void validateNamedDefinition(TransportVersionDefinition definition, Set<
             }
         }
         // validate any modifications
-        TransportVersionDefinition originalDefinition = getResources().get().getReferableDefinitionFromUpstream(definition.name());
+        TransportVersionDefinition originalDefinition = getResources().get().getReferableDefinitionFromGitBase(definition.name());
         if (originalDefinition != null) {
             validateIdenticalPrimaryId(definition, originalDefinition);
             for (int i = 1; i < originalDefinition.ids().size(); ++i) {
@@ -178,7 +178,7 @@ private void validateNamedDefinition(TransportVersionDefinition definition, Set<
     }
 
     private void validateUnreferableDefinition(TransportVersionDefinition definition) {
-        TransportVersionDefinition originalDefinition = getResources().get().getUnreferableDefinitionFromUpstream(definition.name());
+        TransportVersionDefinition originalDefinition = getResources().get().getUnreferableDefinitionFromGitBase(definition.name());
         if (originalDefinition != null) {
             validateIdenticalPrimaryId(definition, originalDefinition);
         }
@@ -240,7 +240,7 @@ private void validateUpperBound(
             );
         }
 
-        TransportVersionUpperBound existingUpperBound = getResources().get().getUpperBoundFromUpstream(upperBound.name());
+        TransportVersionUpperBound existingUpperBound = getResources().get().getUpperBoundFromGitBase(upperBound.name());
         if (existingUpperBound != null && getShouldValidatePrimaryIdNotPatch().get()) {
             if (upperBound.definitionId().patch() != 0 && upperBound.definitionId().base() != existingUpperBound.definitionId().base()) {
                 throwUpperBoundFailure(