Update BWC version logic to support multiple bugfix versions (#117943) (#118116) (#118119)

Author: mark-vieira

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.27", "8.16.2"]
+            BWC_VERSION: ["7.17.27", "8.15.6", "8.16.2"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic.yml

@@ -733,7 +733,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.27", "8.16.2"]
+            BWC_VERSION: ["7.17.27", "8.15.6", "8.16.2"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -781,7 +781,7 @@ steps:
               - openjdk21
               - openjdk22
               - openjdk23
-            BWC_VERSION: ["7.17.27", "8.16.2"]
+            BWC_VERSION: ["7.17.27", "8.15.6", "8.16.2"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/snapshotBwcVersions

@@ -1,3 +1,4 @@
 BWC_VERSION:
   - "7.17.27"
+  - "8.15.6"
   - "8.16.2"

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy

@@ -9,9 +9,10 @@
 
 package org.elasticsearch.gradle.internal
 
+import spock.lang.Unroll
+
 import org.elasticsearch.gradle.fixtures.AbstractGitAwareGradleFuncTest
 import org.gradle.testkit.runner.TaskOutcome
-import spock.lang.Unroll
 
 class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleFuncTest {
 
@@ -22,9 +23,11 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
         buildFile << """
             apply plugin: 'elasticsearch.internal-distribution-bwc-setup'
         """
-        execute("git branch origin/8.0", file("cloned"))
+        execute("git branch origin/8.x", file("cloned"))
+        execute("git branch origin/8.3", file("cloned"))
+        execute("git branch origin/8.2", file("cloned"))
+        execute("git branch origin/8.1", file("cloned"))
         execute("git branch origin/7.16", file("cloned"))
-        execute("git branch origin/7.15", file("cloned"))
     }
 
     def "builds distribution from branches via archives extractedAssemble"() {
@@ -48,10 +51,11 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
         assertOutputContains(result.output, "[$bwcDistVersion] > Task :distribution:archives:darwin-tar:${expectedAssembleTaskName}")
 
         where:
-        bwcDistVersion | bwcProject | expectedAssembleTaskName
-        "8.0.0"        | "minor"    | "extractedAssemble"
-        "7.16.0"       | "staged"   | "extractedAssemble"
-        "7.15.2"       | "bugfix"   | "extractedAssemble"
+        bwcDistVersion | bwcProject    | expectedAssembleTaskName
+        "8.4.0"        | "minor"       | "extractedAssemble"
+        "8.3.0"        | "staged"      | "extractedAssemble"
+        "8.2.1"        | "bugfix"      | "extractedAssemble"
+        "8.1.3"        | "bugfix2"     | "extractedAssemble"
     }
 
     @Unroll
@@ -70,8 +74,8 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
 
         where:
         bwcDistVersion | platform
-        "8.0.0"       | "darwin"
-        "8.0.0"       | "linux"
+        "8.4.0"       | "darwin"
+        "8.4.0"       | "linux"
     }
 
     def "bwc expanded distribution folder can be resolved as bwc project artifact"() {
@@ -107,11 +111,11 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
         result.task(":resolveExpandedDistribution").outcome == TaskOutcome.SUCCESS
         result.task(":distribution:bwc:minor:buildBwcDarwinTar").outcome == TaskOutcome.SUCCESS
         and: "assemble task triggered"
-        result.output.contains("[8.0.0] > Task :distribution:archives:darwin-tar:extractedAssemble")
-        result.output.contains("expandedRootPath /distribution/bwc/minor/build/bwc/checkout-8.0/" +
+        result.output.contains("[8.4.0] > Task :distribution:archives:darwin-tar:extractedAssemble")
+        result.output.contains("expandedRootPath /distribution/bwc/minor/build/bwc/checkout-8.x/" +
                         "distribution/archives/darwin-tar/build/install")
-        result.output.contains("nested folder /distribution/bwc/minor/build/bwc/checkout-8.0/" +
-                        "distribution/archives/darwin-tar/build/install/elasticsearch-8.0.0-SNAPSHOT")
+        result.output.contains("nested folder /distribution/bwc/minor/build/bwc/checkout-8.x/" +
+                        "distribution/archives/darwin-tar/build/install/elasticsearch-8.4.0-SNAPSHOT")
     }
 
 }

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy

@@ -57,7 +57,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
 
             elasticsearch_distributions {
               test_distro {
-                  version = "8.0.0"
+                  version = "8.4.0"
                   type = "archive"
                   platform = "linux"
                   architecture = Architecture.current();
@@ -87,7 +87,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
 
             elasticsearch_distributions {
               test_distro {
-                  version = "8.0.0"
+                  version = "8.4.0"
                   type = "archive"
                   platform = "linux"
                   architecture = Architecture.current();

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy

@@ -39,6 +39,8 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
 
     def "yamlRestTestVxCompatTest does nothing when there are no tests"() {
         given:
+        internalBuild()
+
         subProject(":distribution:bwc:maintenance") << """
         configurations { checkout }
         artifacts {
@@ -47,9 +49,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
         """
 
         buildFile << """
-        plugins {
-          id 'elasticsearch.legacy-yaml-rest-compat-test'
-        }
+        apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
         """
 
         when:
@@ -62,7 +62,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
         result.task(transformTask).outcome == TaskOutcome.NO_SOURCE
     }
 
-    def "yamlRestTestVxCompatTest executes and copies api and transforms tests from :bwc:maintenance"() {
+    def "yamlRestCompatTest executes and copies api and transforms tests from :bwc:maintenance"() {
         given:
         internalBuild()
 
@@ -144,6 +144,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
 
     def "yamlRestTestVxCompatTest is wired into check and checkRestCompat"() {
         given:
+        internalBuild()
         withVersionCatalogue()
         subProject(":distribution:bwc:maintenance") << """
         configurations { checkout }
@@ -153,10 +154,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
         """
 
         buildFile << """
-        plugins {
-          id 'elasticsearch.legacy-yaml-rest-compat-test'
-        }
-
+        apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
         """
 
         when:

build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/bwc/bugfix2/build.gradle

@@ -10,9 +10,11 @@
 rootProject.name = "root"
 
 include ":distribution:bwc:bugfix"
+include ":distribution:bwc:bugfix2"
 include ":distribution:bwc:minor"
 include ":distribution:bwc:major"
 include ":distribution:bwc:staged"
+include ":distribution:bwc:maintenance"
 include ":distribution:archives:darwin-tar"
 include ":distribution:archives:oss-darwin-tar"
 include ":distribution:archives:darwin-aarch64-tar"

build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/bwc/maintenance/build.gradle

@@ -23,15 +23,16 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.TreeMap;
-import java.util.TreeSet;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import static java.util.Collections.reverseOrder;
 import static java.util.Collections.unmodifiableList;
+import static java.util.Comparator.comparing;
 
 /**
  * A container for elasticsearch supported version information used in BWC testing.
@@ -70,18 +71,17 @@ public class BwcVersions implements Serializable {
     private static final Pattern LINE_PATTERN = Pattern.compile(
         "\\W+public static final Version V_(\\d+)_(\\d+)_(\\d+)(_alpha\\d+|_beta\\d+|_rc\\d+)?.*\\);"
     );
-    private static final Version MINIMUM_WIRE_COMPATIBLE_VERSION = Version.fromString("7.17.0");
     private static final String GLIBC_VERSION_ENV_VAR = "GLIBC_VERSION";
 
     private final Version currentVersion;
     private final transient List<Version> versions;
     private final Map<Version, UnreleasedVersionInfo> unreleased;
 
-    public BwcVersions(List<String> versionLines) {
-        this(versionLines, Version.fromString(VersionProperties.getElasticsearch()));
+    public BwcVersions(List<String> versionLines, List<String> developmentBranches) {
+        this(versionLines, Version.fromString(VersionProperties.getElasticsearch()), developmentBranches);
     }
 
-    public BwcVersions(Version currentVersionProperty, List<Version> allVersions) {
+    public BwcVersions(Version currentVersionProperty, List<Version> allVersions, List<String> developmentBranches) {
         if (allVersions.isEmpty()) {
             throw new IllegalArgumentException("Could not parse any versions");
         }
@@ -90,12 +90,12 @@ public BwcVersions(Version currentVersionProperty, List<Version> allVersions) {
         this.currentVersion = allVersions.get(allVersions.size() - 1);
         assertCurrentVersionMatchesParsed(currentVersionProperty);
 
-        this.unreleased = computeUnreleased();
+        this.unreleased = computeUnreleased(developmentBranches);
     }
 
     // Visible for testing
-    BwcVersions(List<String> versionLines, Version currentVersionProperty) {
-        this(currentVersionProperty, parseVersionLines(versionLines));
+    BwcVersions(List<String> versionLines, Version currentVersionProperty, List<String> developmentBranches) {
+        this(currentVersionProperty, parseVersionLines(versionLines), developmentBranches);
     }
 
     private static List<Version> parseVersionLines(List<String> versionLines) {
@@ -132,51 +132,77 @@ public void forPreviousUnreleased(Consumer<UnreleasedVersionInfo> consumer) {
         ).stream().map(unreleased::get).forEach(consumer);
     }
 
-    private String getBranchFor(Version version) {
-        if (version.equals(currentVersion)) {
-            // Just assume the current branch is 'main'. It's actually not important, we never check out the current branch.
-            return "main";
-        } else {
+    private String getBranchFor(Version version, List<String> developmentBranches) {
+        // If the current version matches a specific feature freeze branch, use that
+        if (developmentBranches.contains(version.getMajor() + "." + version.getMinor())) {
             return version.getMajor() + "." + version.getMinor();
+        } else if (developmentBranches.contains(version.getMajor() + ".x")) { // Otherwise if an n.x branch exists and we are that major
+            return version.getMajor() + ".x";
+        } else { // otherwise we're the main branch
+            return "main";
         }
     }
 
-    private Map<Version, UnreleasedVersionInfo> computeUnreleased() {
-        Set<Version> unreleased = new TreeSet<>();
-        // The current version is being worked, is always unreleased
-        unreleased.add(currentVersion);
-        // Recurse for all unreleased versions starting from the current version
-        addUnreleased(unreleased, currentVersion, 0);
+    private Map<Version, UnreleasedVersionInfo> computeUnreleased(List<String> developmentBranches) {
+        Map<Version, UnreleasedVersionInfo> result = new TreeMap<>();
 
-        // Grab the latest version from the previous major if necessary as well, this is going to be a maintenance release
-        Version maintenance = versions.stream()
-            .filter(v -> v.getMajor() == currentVersion.getMajor() - 1)
-            .max(Comparator.naturalOrder())
-            .orElseThrow();
-        // This is considered the maintenance release only if we haven't yet encountered it
-        boolean hasMaintenanceRelease = unreleased.add(maintenance);
+        // The current version is always in development
+        String currentBranch = getBranchFor(currentVersion, developmentBranches);
+        result.put(currentVersion, new UnreleasedVersionInfo(currentVersion, currentBranch, ":distribution"));
+
+        // Check for an n.x branch as well
+        if (currentBranch.equals("main") && developmentBranches.stream().anyMatch(s -> s.endsWith(".x"))) {
+            // This should correspond to the latest new minor
+            Version version = versions.stream()
+                .sorted(Comparator.reverseOrder())
+                .filter(v -> v.getMajor() == (currentVersion.getMajor() - 1) && v.getRevision() == 0)
+                .findFirst()
+                .orElseThrow(() -> new IllegalStateException("Unable to determine development version for branch"));
+            String branch = getBranchFor(version, developmentBranches);
+            assert branch.equals(currentVersion.getMajor() - 1 + ".x") : "Expected branch does not match development branch";
+
+            result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:minor"));
+        }
 
-        List<Version> unreleasedList = unreleased.stream().sorted(Comparator.reverseOrder()).toList();
-        Map<Version, UnreleasedVersionInfo> result = new TreeMap<>();
-        for (int i = 0; i < unreleasedList.size(); i++) {
-            Version esVersion = unreleasedList.get(i);
-            // This is either a new minor or staged release
-            if (currentVersion.equals(esVersion)) {
-                result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution"));
-            } else if (esVersion.getRevision() == 0) {
-                // If there are two upcoming unreleased minors then this one is the new minor
-                if (unreleasedList.get(i + 1).getRevision() == 0) {
-                    result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution:bwc:minor"));
-                } else {
-                    result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution:bwc:staged"));
-                }
-            } else {
-                // If this is the oldest unreleased version and we have a maintenance release
-                if (i == unreleasedList.size() - 1 && hasMaintenanceRelease) {
-                    result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution:bwc:maintenance"));
-                } else {
-                    result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution:bwc:bugfix"));
-                }
+        // Now handle all the feature freeze branches
+        List<String> featureFreezeBranches = developmentBranches.stream()
+            .filter(b -> Pattern.matches("[0-9]+\\.[0-9]+", b))
+            .sorted(reverseOrder(comparing(s -> Version.fromString(s, Version.Mode.RELAXED))))
+            .toList();
+
+        boolean existingBugfix = false;
+        for (int i = 0; i < featureFreezeBranches.size(); i++) {
+            String branch = featureFreezeBranches.get(i);
+            Version version = versions.stream()
+                .sorted(Comparator.reverseOrder())
+                .filter(v -> v.toString().startsWith(branch))
+                .findFirst()
+                .orElse(null);
+
+            // If we don't know about this version we can ignore it
+            if (version == null) {
+                continue;
+            }
+
+            // If this is the current version we can ignore as we've already handled it
+            if (version.equals(currentVersion)) {
+                continue;
+            }
+
+            // We only maintain compatibility back one major so ignore anything older
+            if (currentVersion.getMajor() - version.getMajor() > 1) {
+                continue;
+            }
+
+            // This is the maintenance version
+            if (i == featureFreezeBranches.size() - 1) {
+                result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:maintenance"));
+            } else if (version.getRevision() == 0) { // This is the next staged minor
+                result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:staged"));
+            } else { // This is a bugfix
+                String project = existingBugfix ? "bugfix2" : "bugfix";
+                result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:" + project));
+                existingBugfix = true;
             }
         }
 
@@ -225,7 +251,10 @@ public void compareToAuthoritative(List<Version> authoritativeReleasedVersions)
     }
 
     private List<Version> getReleased() {
-        return versions.stream().filter(v -> unreleased.containsKey(v) == false).toList();
+        return versions.stream()
+            .filter(v -> v.getMajor() >= currentVersion.getMajor() - 1)
+            .filter(v -> unreleased.containsKey(v) == false)
+            .toList();
     }
 
     /**
@@ -251,7 +280,7 @@ public void withIndexCompatible(Predicate<Version> filter, BiConsumer<Version, S
     }
 
     public List<Version> getWireCompatible() {
-        return filterSupportedVersions(versions.stream().filter(v -> v.compareTo(MINIMUM_WIRE_COMPATIBLE_VERSION) >= 0).toList());
+        return filterSupportedVersions(versions.stream().filter(v -> v.compareTo(getMinimumWireCompatibleVersion()) >= 0).toList());
     }
 
     public void withWireCompatible(BiConsumer<Version, String> versionAction) {
@@ -289,7 +318,17 @@ public List<Version> getUnreleasedWireCompatible() {
     }
 
     public Version getMinimumWireCompatibleVersion() {
-        return MINIMUM_WIRE_COMPATIBLE_VERSION;
+        // Determine minimum wire compatible version from list of known versions.
+        // Current BWC policy states the minimum wire compatible version is the last minor release or the previous major version.
+        return versions.stream()
+            .filter(v -> v.getRevision() == 0)
+            .filter(v -> v.getMajor() == currentVersion.getMajor() - 1)
+            .max(Comparator.naturalOrder())
+            .orElseThrow(() -> new IllegalStateException("Unable to determine minimum wire compatible version."));
+    }
+
+    public Version getCurrentVersion() {
+        return currentVersion;
     }
 
     public record UnreleasedVersionInfo(Version version, String branch, String gradleProjectPath) {}