Skip to content

Commit 9deee7e

Browse files
Mikep86Mikep86
committed
Merge branch 'main' into inference_metadata_fields
2 parents d1b8d61 + fd81c51 commit 9deee7e

File tree

609 files changed

+13007
-3750
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

609 files changed

+13007
-3750
lines changed

‎.buildkite/pipelines/intake.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ steps:
5656
timeout_in_minutes: 300
5757
matrix:
5858
setup:
59-
BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"]
59+
BWC_VERSION: ["8.15.6", "8.16.2", "8.17.0", "8.18.0", "9.0.0"]
6060
agents:
6161
provider: gcp
6262
image: family/elasticsearch-ubuntu-2004

‎.buildkite/pipelines/periodic.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -448,7 +448,7 @@ steps:
448448
setup:
449449
ES_RUNTIME_JAVA:
450450
- openjdk21
451-
BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"]
451+
BWC_VERSION: ["8.15.6", "8.16.2", "8.17.0", "8.18.0", "9.0.0"]
452452
agents:
453453
provider: gcp
454454
image: family/elasticsearch-ubuntu-2004
@@ -490,7 +490,7 @@ steps:
490490
ES_RUNTIME_JAVA:
491491
- openjdk21
492492
- openjdk23
493-
BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"]
493+
BWC_VERSION: ["8.15.6", "8.16.2", "8.17.0", "8.18.0", "9.0.0"]
494494
agents:
495495
provider: gcp
496496
image: family/elasticsearch-ubuntu-2004

‎.ci/snapshotBwcVersions‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
BWC_VERSION:
2+
- "8.15.6"
23
- "8.16.2"
34
- "8.17.0"
45
- "8.18.0"

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,10 @@
99

1010
package org.elasticsearch.gradle.internal
1111

12+
import spock.lang.Unroll
13+
1214
import org.elasticsearch.gradle.fixtures.AbstractGitAwareGradleFuncTest
1315
import org.gradle.testkit.runner.TaskOutcome
14-
import spock.lang.Unroll
1516

1617
class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleFuncTest {
1718

@@ -23,8 +24,10 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
2324
apply plugin: 'elasticsearch.internal-distribution-bwc-setup'
2425
"""
2526
execute("git branch origin/8.x", file("cloned"))
27+
execute("git branch origin/8.3", file("cloned"))
28+
execute("git branch origin/8.2", file("cloned"))
29+
execute("git branch origin/8.1", file("cloned"))
2630
execute("git branch origin/7.16", file("cloned"))
27-
execute("git branch origin/7.15", file("cloned"))
2831
}
2932

3033
def "builds distribution from branches via archives extractedAssemble"() {
@@ -48,10 +51,11 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
4851
assertOutputContains(result.output, "[$bwcDistVersion] > Task :distribution:archives:darwin-tar:${expectedAssembleTaskName}")
4952

5053
where:
51-
bwcDistVersion | bwcProject | expectedAssembleTaskName
52-
"8.0.0" | "minor" | "extractedAssemble"
53-
"7.16.0" | "staged" | "extractedAssemble"
54-
"7.15.2" | "bugfix" | "extractedAssemble"
54+
bwcDistVersion | bwcProject | expectedAssembleTaskName
55+
"8.4.0" | "minor" | "extractedAssemble"
56+
"8.3.0" | "staged" | "extractedAssemble"
57+
"8.2.1" | "bugfix" | "extractedAssemble"
58+
"8.1.3" | "bugfix2" | "extractedAssemble"
5559
}
5660

5761
@Unroll
@@ -70,8 +74,8 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
7074

7175
where:
7276
bwcDistVersion | platform
73-
"8.0.0" | "darwin"
74-
"8.0.0" | "linux"
77+
"8.4.0" | "darwin"
78+
"8.4.0" | "linux"
7579
}
7680

7781
def "bwc expanded distribution folder can be resolved as bwc project artifact"() {
@@ -107,11 +111,11 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
107111
result.task(":resolveExpandedDistribution").outcome == TaskOutcome.SUCCESS
108112
result.task(":distribution:bwc:minor:buildBwcDarwinTar").outcome == TaskOutcome.SUCCESS
109113
and: "assemble task triggered"
110-
result.output.contains("[8.0.0] > Task :distribution:archives:darwin-tar:extractedAssemble")
114+
result.output.contains("[8.4.0] > Task :distribution:archives:darwin-tar:extractedAssemble")
111115
result.output.contains("expandedRootPath /distribution/bwc/minor/build/bwc/checkout-8.x/" +
112116
"distribution/archives/darwin-tar/build/install")
113117
result.output.contains("nested folder /distribution/bwc/minor/build/bwc/checkout-8.x/" +
114-
"distribution/archives/darwin-tar/build/install/elasticsearch-8.0.0-SNAPSHOT")
118+
"distribution/archives/darwin-tar/build/install/elasticsearch-8.4.0-SNAPSHOT")
115119
}
116120

117121
}

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
5757
5858
elasticsearch_distributions {
5959
test_distro {
60-
version = "8.0.0"
60+
version = "8.4.0"
6161
type = "archive"
6262
platform = "linux"
6363
architecture = Architecture.current();
@@ -87,7 +87,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
8787
8888
elasticsearch_distributions {
8989
test_distro {
90-
version = "8.0.0"
90+
version = "8.4.0"
9191
type = "archive"
9292
platform = "linux"
9393
architecture = Architecture.current();

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
4040
given:
4141
internalBuild()
4242

43-
subProject(":distribution:bwc:staged") << """
43+
subProject(":distribution:bwc:minor") << """
4444
configurations { checkout }
4545
artifacts {
4646
checkout(new File(projectDir, "checkoutDir"))
@@ -61,11 +61,11 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
6161
result.task(transformTask).outcome == TaskOutcome.NO_SOURCE
6262
}
6363

64-
def "yamlRestCompatTest executes and copies api and transforms tests from :bwc:staged"() {
64+
def "yamlRestCompatTest executes and copies api and transforms tests from :bwc:minor"() {
6565
given:
6666
internalBuild()
6767

68-
subProject(":distribution:bwc:staged") << """
68+
subProject(":distribution:bwc:minor") << """
6969
configurations { checkout }
7070
artifacts {
7171
checkout(new File(projectDir, "checkoutDir"))
@@ -98,8 +98,8 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
9898
String api = "foo.json"
9999
String test = "10_basic.yml"
100100
//add the compatible test and api files, these are the prior version's normal yaml rest tests
101-
file("distribution/bwc/staged/checkoutDir/rest-api-spec/src/main/resources/rest-api-spec/api/" + api) << ""
102-
file("distribution/bwc/staged/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/" + test) << ""
101+
file("distribution/bwc/minor/checkoutDir/rest-api-spec/src/main/resources/rest-api-spec/api/" + api) << ""
102+
file("distribution/bwc/minor/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/" + test) << ""
103103

104104
when:
105105
def result = gradleRunner("yamlRestCompatTest").build()
@@ -145,7 +145,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
145145
given:
146146
internalBuild()
147147
withVersionCatalogue()
148-
subProject(":distribution:bwc:staged") << """
148+
subProject(":distribution:bwc:minor") << """
149149
configurations { checkout }
150150
artifacts {
151151
checkout(new File(projectDir, "checkoutDir"))
@@ -186,7 +186,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
186186
given:
187187
internalBuild()
188188

189-
subProject(":distribution:bwc:staged") << """
189+
subProject(":distribution:bwc:minor") << """
190190
configurations { checkout }
191191
artifacts {
192192
checkout(new File(projectDir, "checkoutDir"))
@@ -230,7 +230,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
230230

231231
setupRestResources([], [])
232232

233-
file("distribution/bwc/staged/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/test.yml" ) << """
233+
file("distribution/bwc/minor/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/test.yml" ) << """
234234
"one":
235235
- do:
236236
do_.some.key_to_replace:

‎build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/bwc/bugfix2/build.gradle‎

Whitespace-only changes.

‎build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/bwc/maintenance/build.gradle‎

Whitespace-only changes.

‎build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/settings.gradle‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,11 @@
1010
rootProject.name = "root"
1111

1212
include ":distribution:bwc:bugfix"
13+
include ":distribution:bwc:bugfix2"
1314
include ":distribution:bwc:minor"
1415
include ":distribution:bwc:major"
1516
include ":distribution:bwc:staged"
17+
include ":distribution:bwc:maintenance"
1618
include ":distribution:archives:darwin-tar"
1719
include ":distribution:archives:oss-darwin-tar"
1820
include ":distribution:archives:darwin-aarch64-tar"

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java‎

Lines changed: 73 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,15 @@
2121
import java.util.Optional;
2222
import java.util.Set;
2323
import java.util.TreeMap;
24-
import java.util.TreeSet;
2524
import java.util.function.BiConsumer;
2625
import java.util.function.Consumer;
2726
import java.util.function.Predicate;
2827
import java.util.regex.Matcher;
2928
import java.util.regex.Pattern;
3029

30+
import static java.util.Collections.reverseOrder;
3131
import static java.util.Collections.unmodifiableList;
32+
import static java.util.Comparator.comparing;
3233

3334
/**
3435
* A container for elasticsearch supported version information used in BWC testing.
@@ -73,11 +74,11 @@ public class BwcVersions implements Serializable {
7374
private final transient List<Version> versions;
7475
private final Map<Version, UnreleasedVersionInfo> unreleased;
7576

76-
public BwcVersions(List<String> versionLines) {
77-
this(versionLines, Version.fromString(VersionProperties.getElasticsearch()));
77+
public BwcVersions(List<String> versionLines, List<String> developmentBranches) {
78+
this(versionLines, Version.fromString(VersionProperties.getElasticsearch()), developmentBranches);
7879
}
7980

80-
public BwcVersions(Version currentVersionProperty, List<Version> allVersions) {
81+
public BwcVersions(Version currentVersionProperty, List<Version> allVersions, List<String> developmentBranches) {
8182
if (allVersions.isEmpty()) {
8283
throw new IllegalArgumentException("Could not parse any versions");
8384
}
@@ -86,12 +87,12 @@ public BwcVersions(Version currentVersionProperty, List<Version> allVersions) {
8687
this.currentVersion = allVersions.get(allVersions.size() - 1);
8788
assertCurrentVersionMatchesParsed(currentVersionProperty);
8889

89-
this.unreleased = computeUnreleased();
90+
this.unreleased = computeUnreleased(developmentBranches);
9091
}
9192

9293
// Visible for testing
93-
BwcVersions(List<String> versionLines, Version currentVersionProperty) {
94-
this(currentVersionProperty, parseVersionLines(versionLines));
94+
BwcVersions(List<String> versionLines, Version currentVersionProperty, List<String> developmentBranches) {
95+
this(currentVersionProperty, parseVersionLines(versionLines), developmentBranches);
9596
}
9697

9798
private static List<Version> parseVersionLines(List<String> versionLines) {
@@ -126,58 +127,77 @@ public void forPreviousUnreleased(Consumer<UnreleasedVersionInfo> consumer) {
126127
getUnreleased().stream().filter(version -> version.equals(currentVersion) == false).map(unreleased::get).forEach(consumer);
127128
}
128129

129-
private String getBranchFor(Version version) {
130-
if (version.equals(currentVersion)) {
131-
// Just assume the current branch is 'main'. It's actually not important, we never check out the current branch.
132-
return "main";
133-
} else {
130+
private String getBranchFor(Version version, List<String> developmentBranches) {
131+
// If the current version matches a specific feature freeze branch, use that
132+
if (developmentBranches.contains(version.getMajor() + "." + version.getMinor())) {
134133
return version.getMajor() + "." + version.getMinor();
134+
} else if (developmentBranches.contains(version.getMajor() + ".x")) { // Otherwise if an n.x branch exists and we are that major
135+
return version.getMajor() + ".x";
136+
} else { // otherwise we're the main branch
137+
return "main";
135138
}
136139
}
137140

138-
private Map<Version, UnreleasedVersionInfo> computeUnreleased() {
139-
Set<Version> unreleased = new TreeSet<>();
140-
// The current version is being worked, is always unreleased
141-
unreleased.add(currentVersion);
142-
// Recurse for all unreleased versions starting from the current version
143-
addUnreleased(unreleased, currentVersion, 0);
141+
private Map<Version, UnreleasedVersionInfo> computeUnreleased(List<String> developmentBranches) {
142+
Map<Version, UnreleasedVersionInfo> result = new TreeMap<>();
144143

145-
// Grab the latest version from the previous major if necessary as well, this is going to be a maintenance release
146-
Version maintenance = versions.stream()
147-
.filter(v -> v.getMajor() == currentVersion.getMajor() - 1)
148-
.max(Comparator.naturalOrder())
149-
.orElseThrow();
150-
// This is considered the maintenance release only if we haven't yet encountered it
151-
boolean hasMaintenanceRelease = unreleased.add(maintenance);
144+
// The current version is always in development
145+
String currentBranch = getBranchFor(currentVersion, developmentBranches);
146+
result.put(currentVersion, new UnreleasedVersionInfo(currentVersion, currentBranch, ":distribution"));
147+
148+
// Check for an n.x branch as well
149+
if (currentBranch.equals("main") && developmentBranches.stream().anyMatch(s -> s.endsWith(".x"))) {
150+
// This should correspond to the latest new minor
151+
Version version = versions.stream()
152+
.sorted(Comparator.reverseOrder())
153+
.filter(v -> v.getMajor() == (currentVersion.getMajor() - 1) && v.getRevision() == 0)
154+
.findFirst()
155+
.orElseThrow(() -> new IllegalStateException("Unable to determine development version for branch"));
156+
String branch = getBranchFor(version, developmentBranches);
157+
assert branch.equals(currentVersion.getMajor() - 1 + ".x") : "Expected branch does not match development branch";
158+
159+
result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:minor"));
160+
}
152161

153-
List<Version> unreleasedList = unreleased.stream().sorted(Comparator.reverseOrder()).toList();
154-
Map<Version, UnreleasedVersionInfo> result = new TreeMap<>();
155-
boolean newMinor = false;
156-
for (int i = 0; i < unreleasedList.size(); i++) {
157-
Version esVersion = unreleasedList.get(i);
158-
// This is either a new minor or staged release
159-
if (currentVersion.equals(esVersion)) {
160-
result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution"));
161-
} else if (esVersion.getRevision() == 0) {
162-
// If there are two upcoming unreleased minors then this one is the new minor
163-
if (newMinor == false && unreleasedList.get(i + 1).getRevision() == 0) {
164-
result.put(esVersion, new UnreleasedVersionInfo(esVersion, esVersion.getMajor() + ".x", ":distribution:bwc:minor"));
165-
newMinor = true;
166-
} else if (newMinor == false
167-
&& unreleasedList.stream().filter(v -> v.getMajor() == esVersion.getMajor() && v.getRevision() == 0).count() == 1) {
168-
// This is the only unreleased new minor which means we've not yet staged it for release
169-
result.put(esVersion, new UnreleasedVersionInfo(esVersion, esVersion.getMajor() + ".x", ":distribution:bwc:minor"));
170-
newMinor = true;
171-
} else {
172-
result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution:bwc:staged"));
173-
}
174-
} else {
175-
// If this is the oldest unreleased version and we have a maintenance release
176-
if (i == unreleasedList.size() - 1 && hasMaintenanceRelease) {
177-
result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution:bwc:maintenance"));
178-
} else {
179-
result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution:bwc:bugfix"));
180-
}
162+
// Now handle all the feature freeze branches
163+
List<String> featureFreezeBranches = developmentBranches.stream()
164+
.filter(b -> Pattern.matches("[0-9]+\\.[0-9]+", b))
165+
.sorted(reverseOrder(comparing(s -> Version.fromString(s, Version.Mode.RELAXED))))
166+
.toList();
167+
168+
boolean existingBugfix = false;
169+
for (int i = 0; i < featureFreezeBranches.size(); i++) {
170+
String branch = featureFreezeBranches.get(i);
171+
Version version = versions.stream()
172+
.sorted(Comparator.reverseOrder())
173+
.filter(v -> v.toString().startsWith(branch))
174+
.findFirst()
175+
.orElse(null);
176+
177+
// If we don't know about this version we can ignore it
178+
if (version == null) {
179+
continue;
180+
}
181+
182+
// If this is the current version we can ignore as we've already handled it
183+
if (version.equals(currentVersion)) {
184+
continue;
185+
}
186+
187+
// We only maintain compatibility back one major so ignore anything older
188+
if (currentVersion.getMajor() - version.getMajor() > 1) {
189+
continue;
190+
}
191+
192+
// This is the maintenance version
193+
if (i == featureFreezeBranches.size() - 1) {
194+
result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:maintenance"));
195+
} else if (version.getRevision() == 0) { // This is the next staged minor
196+
result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:staged"));
197+
} else { // This is a bugfix
198+
String project = existingBugfix ? "bugfix2" : "bugfix";
199+
result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:" + project));
200+
existingBugfix = true;
181201
}
182202
}
183203

0 commit comments

Comments
 (0)