Skip to content

Commit 0e36d2b

Browse files
JVerwolfJVerwolf
committed
update build tasks
1 parent 5581209 commit 0e36d2b

File tree

7 files changed

+285
-10
lines changed

7 files changed

+285
-10
lines changed

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataPlugin.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,11 @@ public class GenerateTransportVersionDataPlugin implements Plugin<Project> {
1818
public void apply(Project project) {
1919
project.getTasks().register("generateTransportVersionData", GenerateTransportVersionDataTask.class, t -> {
2020
t.setDescription("Generate transport version data"); // todo update this to be more descriptive
21-
22-
});
21+
t.setGroup("Transport Versions"); // todo
22+
t.getDataFileDirectory().set(
23+
project.getLayout().getProjectDirectory().file("src/main/resources/org/elasticsearch/transport/"));
24+
t.getTVSetName().set("test"); //todo
25+
t.getReleaseVersionMajorMinor().set("9.2"); // todo
26+
});
2327
}
2428
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java‎

Lines changed: 177 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,183 @@
99

1010
package org.elasticsearch.gradle.internal.transport;
1111

12+
import com.google.common.collect.Streams;
1213
import org.gradle.api.DefaultTask;
14+
import org.gradle.api.GradleException;
15+
import org.gradle.api.file.RegularFileProperty;
16+
import org.gradle.api.provider.Property;
17+
import org.gradle.api.tasks.Input;
18+
import org.gradle.api.tasks.InputDirectory;
19+
import org.gradle.api.tasks.TaskAction;
1320

14-
public class GenerateTransportVersionDataTask extends DefaultTask {
21+
import java.io.File;
22+
import java.util.Arrays;
23+
import java.util.List;
24+
import java.util.Objects;
25+
import java.util.regex.Pattern;
26+
import java.util.stream.Stream;
27+
28+
import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.formatLatestTVSetFilename;
29+
30+
/**
31+
* This task generates TransportVersionSetData data files that contain information about transport versions. These files
32+
* are added to the server project's resource directory at `server/src/main/resources/org/elasticsearch/transport/`.
33+
* They have the following format:
34+
* <pre>
35+
* Filename: my-transport-version-set.json // Must be the same as the name of the transport version set.
36+
* {
37+
* "name": "my-transport-version-set", // The name of the transport version set used for reference in the code.
38+
* "ids": [
39+
* 9109000, // The transport version introduced to the main branch.
40+
* 8841059 // The transport version backported to a previous release branch.
41+
* ]
42+
* }
43+
* </pre>
44+
*/
45+
public abstract class GenerateTransportVersionDataTask extends DefaultTask {
46+
47+
/**
48+
* Specifies the directory in which contains all TransportVersionSet data files.
49+
*
50+
* @return
51+
*/
52+
@InputDirectory
53+
public abstract RegularFileProperty getDataFileDirectory();
54+
55+
/**
56+
* Used to set the name of the TransportVersionSet for which a data file will be generated.
57+
*/
58+
@Input
59+
public abstract Property<String> getTVSetName();
60+
61+
/**
62+
* Used to set the `major.minor` release version for which the specific TransportVersion ID will be generated.
63+
* E.g.: "9.2", "8.18", etc.
64+
*/
65+
@Input
66+
public abstract Property<String> getReleaseVersionMajorMinor();
67+
68+
69+
@TaskAction
70+
public void generateTransportVersionData() {
71+
var tvDataDir = Objects.requireNonNull(getDataFileDirectory().getAsFile().get());
72+
var newTVName = Objects.requireNonNull(getTVSetName().get());
73+
var majorMinor = Objects.requireNonNull(getReleaseVersionMajorMinor().get());
74+
75+
// Split version into major and minor
76+
String[] versionParts = majorMinor.split("\\.");
77+
assert versionParts.length == 2;
78+
var major = Integer.parseInt(versionParts[0]);
79+
var minor = Integer.parseInt(versionParts[1]);
80+
81+
// Get the latest transport version data from the loc.
82+
var latestTVSetData = TransportVersionUtils.getLatestTVSetData(tvDataDir, majorMinor);
83+
84+
// Get the latest transport version data for the prior release version.
85+
var priorLatestTVSetDataFileName = getPriorLatestTVSetFilename(tvDataDir, majorMinor);
86+
var priorLatestTVSetData = TransportVersionUtils.getLatestTVSetData(tvDataDir, priorLatestTVSetDataFileName);
87+
if (priorLatestTVSetData == null) {
88+
// TODO Can this ever be null? No, must populate the data file for the latest branch we can no longer backport to.
89+
}
90+
91+
// Bump the version number
92+
int nextVersion;
93+
if (latestTVSetData == null) {
94+
// TODO do a major or minor version bump here
95+
if (minor == 0) {
96+
// This is major bump
97+
nextVersion = major * 1_000_000;
98+
} else {
99+
// This is a minor bump. Just increment as usual but from the prior version.
100+
assert priorLatestTVSetData != null;
101+
nextVersion = bumpVersionNumber(priorLatestTVSetData.ids.getFirst());
102+
}
103+
} else {
104+
nextVersion = bumpVersionNumber(latestTVSetData.ids.getFirst());
105+
}
106+
System.out.println("Latest transport version set: " + latestTVSetData.name + " with IDs: " + latestTVSetData.ids);
107+
108+
109+
// Load the tvSetData for the specified name.
110+
var tvSetDataFromFile = TransportVersionUtils.getTVSetData(tvDataDir, newTVName);
111+
112+
// Create/update the data files
113+
if (tvSetDataFromFile == null) {
114+
// Create a new data file for the case where this is a new TV
115+
new TransportVersionUtils.TransportVersionSetData(newTVName, List.of(nextVersion)).writeToDataDir(tvDataDir);
116+
} else {
117+
// This is not a new TV. We are backporting an existing TVSet.
118+
// Check to ensure that there isn't already a TV number for this change (e.g. if this task has been run twice).
119+
var existingIDsForReleaseVersion = tvSetDataFromFile.ids.stream().filter(id -> {
120+
var priorLatestID = priorLatestTVSetData.ids.getFirst();
121+
var latestID = latestTVSetData.ids.getFirst();
122+
return priorLatestID < id && id < latestID;
123+
}).toList();
124+
if (existingIDsForReleaseVersion.isEmpty() == false) {
125+
throw new GradleException("TransportVersion already exists for this release! Release version: " +
126+
majorMinor + "TransportVersion Id: " + existingIDsForReleaseVersion.stream().findFirst());
127+
}
128+
129+
// Update the existing data file for the backport.
130+
new TransportVersionUtils.TransportVersionSetData(
131+
newTVName,
132+
Streams.concat(tvSetDataFromFile.ids.stream(), Stream.of(nextVersion)).sorted().toList().reversed()
133+
).writeToDataDir(tvDataDir);
134+
}
135+
136+
// Update the LATEST file.
137+
TransportVersionUtils.writeTVSetData(
138+
tvDataDir,
139+
formatLatestTVSetFilename(majorMinor),
140+
new TransportVersionUtils.TransportVersionSetData(newTVName, List.of(nextVersion))
141+
);
142+
}
143+
144+
145+
// TODO account for bumping majors. Need to make a new data file too.
146+
private static int bumpVersionNumber(int versionNumber) {
147+
var main = false; // TODO how do we know if we are on main?
148+
149+
/*
150+
* M_NNN_S_PP
151+
*
152+
* M - The major version of Elasticsearch
153+
* NNN - The server version part
154+
* S - The subsidiary version part. It should always be 0 here, it is only used in subsidiary repositories.
155+
* PP - The patch version part
156+
*/
157+
if (main) {
158+
// bump the server versin part
159+
return versionNumber + 1000; // TODO add check that this doesn't cause overflow out of server versions
160+
} else {
161+
// bump the patch version part
162+
return versionNumber + 1; // TODO add check that this doesn't cause overflow out of patch versions
163+
}
164+
}
165+
166+
/**
167+
* Accepts a major.minor version string (e.g. "9.0") and returns the LATEST.json file of the
168+
* previous release string (e.g. "8.19-LATEST.json").
169+
*/
170+
private static String getPriorLatestTVSetFilename(File tvDataDir, int major, int minor) {
171+
assert tvDataDir != null;
172+
assert tvDataDir.isDirectory();
173+
174+
if (minor > 0) {
175+
return formatLatestTVSetFilename(major, minor - 1);
176+
}
177+
178+
// If the minor is 0, we need to find the largest minor on the previous major
179+
var pattern = Pattern.compile("^(\\d+)\\.(\\d+)-LATEST\\.json$");
180+
var highestMinorOfPrevMajor = Arrays.stream(Objects.requireNonNull(tvDataDir.listFiles()))
181+
.filter(tvDataFile -> tvDataFile.getName().endsWith("-LATEST.json"))
182+
.flatMap(tvDataFile -> {
183+
var matcher = pattern.matcher(tvDataFile.getName());
184+
var localMajor = Integer.parseInt(matcher.group(1));
185+
var localMinor = Integer.parseInt(matcher.group(2));
186+
return localMajor == major - 1 ? Stream.of(localMinor) : Stream.empty();
187+
}).sorted().toList().getLast();
188+
189+
return formatLatestTVSetFilename(major - 1, highestMinorOfPrevMajor);
190+
}
15191
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsTask.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,22 +36,22 @@
3636
import java.util.Set;
3737

3838
/**
39-
* This task locates all method invocations of org.elasticsearch.TransportVersionSet#get(java.lang.String) in the
39+
* This task locates all method invocations of org.elasticsearch.TransportVersionSetData#get(java.lang.String) in the
4040
* provided directory, and then records the value of string literals passed as arguments. It then records each
4141
* String on a newline in the provided output file.
4242
*/
4343
public abstract class LocateTransportVersionsTask extends DefaultTask {
44-
public static final String TRANSPORT_VERSION_SET_CLASS = "org/elasticsearch/TransportVersionSet";
44+
public static final String TRANSPORT_VERSION_SET_CLASS = "org/elasticsearch/TransportVersionSetData";
4545
public static final String TRANSPORT_VERSION_SET_METHOD_NAME = "get";
4646

4747
/**
48-
* The directory to scan for TransportVersionSet#get invocations.
48+
* The directory to scan for TransportVersionSetData#get invocations.
4949
*/
5050
@InputFiles
5151
public abstract Property<FileCollection> getClassDirs();
5252

5353
/**
54-
* The output file, with each newline containing the string literal argument of each TransportVersionSet#get
54+
* The output file, with each newline containing the string literal argument of each TransportVersionSetData#get
5555
* invocation.
5656
*/
5757
@OutputFile

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.gradle.internal.transport;
11+
12+
import com.fasterxml.jackson.databind.ObjectMapper;
13+
14+
import java.io.File;
15+
import java.io.Serializable;
16+
import java.nio.file.Path;
17+
import java.util.ArrayList;
18+
import java.util.List;
19+
20+
public class TransportVersionUtils {
21+
public static final String LATEST_SUFFIX = "-LATEST.json";
22+
public static final String JSON_SUFFIX = ".json";
23+
24+
25+
public static class TransportVersionSetData implements Serializable {
26+
public String name;
27+
public List<Integer> ids;
28+
29+
30+
public TransportVersionSetData(String name, List<Integer> ids) {
31+
this.name = name;
32+
this.ids = ids;
33+
}
34+
35+
public TransportVersionSetData(String name) {
36+
this(name, new ArrayList<>());
37+
}
38+
39+
public void writeToDataDir(File tvDataDir) {
40+
TransportVersionUtils.writeTVSetData(tvDataDir, name, this);
41+
}
42+
}
43+
44+
public static void writeTVSetData(File tvDataDir, String filename, TransportVersionSetData versionSetData) {
45+
ObjectMapper mapper = new ObjectMapper();
46+
try {
47+
File tvSetFile = tvDataDir.toPath().resolve(filename + JSON_SUFFIX).toFile();
48+
mapper.writeValue(tvSetFile, versionSetData);
49+
} catch (Exception e) {
50+
throw new RuntimeException("Failed to write the TransportVersionSet data file: " + tvDataDir.getAbsolutePath(), e);
51+
}
52+
}
53+
54+
55+
public static TransportVersionSetData getLatestTVSetData(File tvDataDir, String majorMinor) {
56+
return getTVSetData(tvDataDir.toPath().resolve(majorMinor + LATEST_SUFFIX));
57+
58+
}
59+
60+
public static TransportVersionSetData getTVSetData(File tvDataDir, String tvSetNameField) {
61+
return getTVSetData(tvDataDir.toPath().resolve(tvSetNameField + JSON_SUFFIX));
62+
}
63+
64+
/**
65+
* Returns the TransportVersionSetData read from the file at the specified path, null if no file exists.
66+
*/
67+
public static TransportVersionSetData getTVSetData(Path path) {
68+
File tvSetDataFile = path.toFile();
69+
if (tvSetDataFile.exists() == false) {
70+
return null;
71+
}
72+
73+
ObjectMapper mapper = new ObjectMapper();
74+
try {
75+
return mapper.readValue(tvSetDataFile, TransportVersionSetData.class);
76+
} catch (Exception e) {
77+
throw new RuntimeException("Failed to read the TransportVersionSet data file: " + tvSetDataFile.getAbsolutePath(), e);
78+
}
79+
}
80+
81+
public static String formatLatestTVSetFilename(int major, int minor) {
82+
return formatLatestTVSetFilename(major + "." + minor);
83+
}
84+
85+
public static String formatLatestTVSetFilename(String majorMinor) {
86+
return majorMinor + LATEST_SUFFIX;
87+
}
88+
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionsPlugin.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ public void apply(Project project) {
2222
var dir = project.getLayout().getProjectDirectory().file("src/main/resources/org/elasticsearch/transport/");
2323
t.getDataFileDirectory().set(dir);
2424
t.getTransportVersionSetNamesFile().set(project.getLayout().getBuildDirectory().file(TRANSPORT_VERSION_NAMES_FILE));
25+
// TODO is this correct? Needs to have both global/per-plugin versions and dependencies
26+
t.dependsOn(project.getTasks().withType(LocateTransportVersionsTask.class));
27+
t.setGroup("Transport Versions"); // TODO
28+
t.setDescription("Validates that the transport versions used in the project are correct and up to date."); // TODO
2529
});
2630
}
2731
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionsTask.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@
2626

2727
/**
2828
* Validates that each transport version declaration has an associated metadata file.
29+
* TODO:
30+
* - make this depend on the LocateTransportVersions task/plugin
31+
* - make this both a local (per module/plugin) and global task
2932
*/
3033
public abstract class ValidateTransportVersionsTask extends DefaultTask {
3134

@@ -59,8 +62,8 @@ public void validateTransportVersions() throws IOException {
5962
try (var reader = new BufferedReader(new FileReader(tvSetDeclaredNamesFile))) {
6063
reader.lines().forEach(declaredName -> {
6164
if (tvSetNamesInDataFiles.contains(declaredName) == false) {
62-
throw new RuntimeException("TransportVersionSet.get(\"" + declaredName + "\") was used, but lacks a" +
63-
"data file with a corresponding transport version. This can be generated with the <TODO> task"); //TODO
65+
throw new RuntimeException("TransportVersionSetData.get(\"" + declaredName + "\") was used, but lacks a" +
66+
" data file with a corresponding transport version. This can be generated with the <TODO> task"); //TODO
6467
}
6568
});
6669
}

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -320,7 +320,7 @@ static TransportVersion def(int id) {
320320
public static final TransportVersion SPARSE_VECTOR_FIELD_PRUNING_OPTIONS = def(9_107_0_00);
321321
public static final TransportVersion CLUSTER_STATE_PROJECTS_SETTINGS = def(9_108_0_00);
322322

323-
public static final TransportVersionSet TEST = TransportVersionSet.get("test-potato");
323+
324324

325325
/*
326326
* STOP! READ THIS FIRST! No, really,

0 commit comments

Comments
 (0)