Skip to content

Commit dc4d436

Browse files
jdconradjdconrad
authored
[9.0] Update transport version directory structure (#132891) (#132993)
This changes the directory structure for transport version files to the following: /transport/latest /transport/definitions/named /transport/definitions/initial /transport/definitions/manifest.txt This allows manifest to be placed in a location for named and initial transport versions without adding confusion to latest. ES-12401
1 parent 3a42af7 commit dc4d436

File tree

15 files changed

+84
-58
lines changed

15 files changed

+84
-58
lines changed

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ testfixtures_shared/
6969
# Generated
7070
checkstyle_ide.xml
7171
x-pack/plugin/esql/src/main/generated-src/generated/
72-
server/src/main/resources/transport/defined/manifest.txt
72+
server/src/main/resources/transport/definitions/manifest.txt
7373

7474
# JEnv
7575
.java-version

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPluginFuncTest.groovy‎

Lines changed: 24 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,12 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
4242
}
4343
}
4444

45-
def definedTransportVersion(String name, String ids) {
46-
javaResource("myserver", "transport/defined/" + name + ".csv", ids)
45+
def namedTransportVersion(String name, String ids) {
46+
javaResource("myserver", "transport/definitions/named/" + name + ".csv", ids)
47+
}
48+
49+
def initialTransportVersion(String name, String id) {
50+
javaResource("myserver", "transport/definitions/initial/" + name + ".csv", id)
4751
}
4852

4953
def definedAndUsedTransportVersion(String name, String ids) {
@@ -54,7 +58,7 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
5458
javaSource("myserver", "org.elasticsearch", classname, "", """
5559
static final TransportVersion usage = TransportVersion.fromName("${name}");
5660
""")
57-
definedTransportVersion(name, ids)
61+
namedTransportVersion(name, ids)
5862
}
5963

6064
def latestTransportVersion(String branch, String name, String id) {
@@ -95,8 +99,9 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
9599
apply plugin: 'elasticsearch.transport-version-references'
96100
apply plugin: 'elasticsearch.transport-version-resources'
97101
"""
98-
definedTransportVersion("existing_91", "8012000")
99-
definedTransportVersion("existing_92", "8123000,8012001")
102+
namedTransportVersion("existing_91", "8012000")
103+
namedTransportVersion("existing_92", "8123000,8012001")
104+
initialTransportVersion("initial_9_0_0", "8000000")
100105
latestTransportVersion("9.2", "existing_92", "8123000")
101106
latestTransportVersion("9.1", "existing_92", "8012001")
102107
// a mock version of TransportVersion, just here so we can compile Dummy.java et al
@@ -148,12 +153,12 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
148153

149154
def "references must be defined"() {
150155
given:
151-
definedTransportVersion("not_used", "1000000")
156+
namedTransportVersion("not_used", "1000000")
152157
when:
153158
def result = validateDefinitionsFails()
154159
then:
155160
assertDefinitionsFailure(result, "Transport version definition file " +
156-
"[myserver/src/main/resources/transport/defined/not_used.csv] is not referenced")
161+
"[myserver/src/main/resources/transport/definitions/named/not_used.csv] is not referenced")
157162
}
158163

159164
def "names must be lowercase alphanum or underscore"() {
@@ -163,7 +168,7 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
163168
def result = validateDefinitionsFails()
164169
then:
165170
assertDefinitionsFailure(result, "Transport version definition file " +
166-
"[myserver/src/main/resources/transport/defined/${name}.csv] does not have a valid name, " +
171+
"[myserver/src/main/resources/transport/definitions/named/${name}.csv] does not have a valid name, " +
167172
"must be lowercase alphanumeric and underscore")
168173

169174
where:
@@ -177,7 +182,7 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
177182
def result = validateDefinitionsFails()
178183
then:
179184
assertDefinitionsFailure(result, "Transport version definition file " +
180-
"[myserver/src/main/resources/transport/defined/empty.csv] does not contain any ids")
185+
"[myserver/src/main/resources/transport/definitions/named/empty.csv] does not contain any ids")
181186
}
182187

183188
def "definitions have ids in descending order"() {
@@ -187,7 +192,7 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
187192
def result = validateDefinitionsFails()
188193
then:
189194
assertDefinitionsFailure(result, "Transport version definition file " +
190-
"[myserver/src/main/resources/transport/defined/out_of_order.csv] does not have ordered ids")
195+
"[myserver/src/main/resources/transport/definitions/named/out_of_order.csv] does not have ordered ids")
191196
}
192197

193198
def "definition ids are unique"() {
@@ -197,8 +202,8 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
197202
def result = validateDefinitionsFails()
198203
then:
199204
assertDefinitionsFailure(result, "Transport version definition file " +
200-
"[myserver/src/main/resources/transport/defined/existing_92.csv] contains id 8123000 already defined in " +
201-
"[myserver/src/main/resources/transport/defined/duplicate.csv]")
205+
"[myserver/src/main/resources/transport/definitions/named/existing_92.csv] contains id 8123000 already defined in " +
206+
"[myserver/src/main/resources/transport/definitions/named/duplicate.csv]")
202207
}
203208

204209
def "definitions have bwc ids with non-zero patch part"() {
@@ -208,27 +213,27 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
208213
def result = validateDefinitionsFails()
209214
then:
210215
assertDefinitionsFailure(result, "Transport version definition file " +
211-
"[myserver/src/main/resources/transport/defined/patched.csv] contains bwc id [8100000] with a patch part of 0")
216+
"[myserver/src/main/resources/transport/definitions/named/patched.csv] contains bwc id [8100000] with a patch part of 0")
212217
}
213218

214219
def "definitions have primary ids which cannot change"() {
215220
given:
216-
definedTransportVersion("existing_92", "8500000")
221+
namedTransportVersion("existing_92", "8500000")
217222
when:
218223
def result = validateDefinitionsFails()
219224
then:
220225
assertDefinitionsFailure(result, "Transport version definition file " +
221-
"[myserver/src/main/resources/transport/defined/existing_92.csv] has modified primary id from 8123000 to 8500000")
226+
"[myserver/src/main/resources/transport/definitions/named/existing_92.csv] has modified primary id from 8123000 to 8500000")
222227
}
223228

224229
def "cannot change committed ids to a branch"() {
225230
given:
226-
definedTransportVersion("existing_92", "8123000,8012002")
231+
namedTransportVersion("existing_92", "8123000,8012002")
227232
when:
228233
def result = validateDefinitionsFails()
229234
then:
230235
assertDefinitionsFailure(result, "Transport version definition file " +
231-
"[myserver/src/main/resources/transport/defined/existing_92.csv] modifies existing patch id from 8012001 to 8012002")
236+
"[myserver/src/main/resources/transport/definitions/named/existing_92.csv] modifies existing patch id from 8012001 to 8012002")
232237
}
233238

234239
def "latest files must reference defined name"() {
@@ -249,7 +254,7 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
249254
then:
250255
assertDefinitionsFailure(result, "Latest transport version file " +
251256
"[myserver/src/main/resources/transport/latest/9.2.csv] has id 8124000 which is not in definition " +
252-
"[myserver/src/main/resources/transport/defined/existing_92.csv]")
257+
"[myserver/src/main/resources/transport/definitions/named/existing_92.csv]")
253258
}
254259

255260
def "latest files have latest id within base"() {
@@ -296,6 +301,6 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
296301
def result = validateDefinitionsFails()
297302
then:
298303
assertDefinitionsFailure(result, "Transport version definition file " +
299-
"[myserver/src/main/resources/transport/defined/patch.csv] has patch version 8015001 as primary id")
304+
"[myserver/src/main/resources/transport/definitions/named/patch.csv] has patch version 8015001 as primary id")
300305
}
301306
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,11 @@
2121

2222
import java.io.IOException;
2323
import java.nio.charset.StandardCharsets;
24+
import java.nio.file.FileVisitResult;
2425
import java.nio.file.Files;
2526
import java.nio.file.Path;
27+
import java.nio.file.SimpleFileVisitor;
28+
import java.nio.file.attribute.BasicFileAttributes;
2629

2730
public abstract class GenerateTransportVersionManifestTask extends DefaultTask {
2831
@InputDirectory
@@ -44,15 +47,14 @@ public void generateTransportVersionManifest() throws IOException {
4447
}
4548

4649
try (var writer = Files.newBufferedWriter(manifestFile)) {
47-
try (var stream = Files.list(definitionsDir)) {
48-
for (String filename : stream.map(p -> p.getFileName().toString()).toList()) {
49-
if (filename.equals(manifestFile.getFileName().toString())) {
50-
// don't list self
51-
continue;
52-
}
53-
writer.write(filename + "\n");
50+
Files.walkFileTree(definitionsDir, new SimpleFileVisitor<>() {
51+
@Override
52+
public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException {
53+
String subPath = definitionsDir.relativize(path).toString().replace('\\', '/');
54+
writer.write(subPath + "\n");
55+
return FileVisitResult.CONTINUE;
5456
}
55-
}
57+
});
5658
}
5759
}
5860
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ public void apply(Project project) {
6464
t.getManifestFile().set(project.getLayout().getBuildDirectory().file("generated-resources/manifest.txt"));
6565
});
6666
project.getTasks().named(JavaPlugin.PROCESS_RESOURCES_TASK_NAME, Copy.class).configure(t -> {
67-
t.into("transport/defined", c -> c.from(generateManifestTask));
67+
t.into("transport/definitions", c -> c.from(generateManifestTask));
6868
});
6969
}
7070
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
class TransportVersionUtils {
2121

2222
static Path definitionFilePath(Directory resourcesDirectory, String name) {
23-
return getDefinitionsDirectory(resourcesDirectory).getAsFile().toPath().resolve(name + ".csv");
23+
return getDefinitionsDirectory(resourcesDirectory).getAsFile().toPath().resolve("named/" + name + ".csv");
2424
}
2525

2626
static Path latestFilePath(Directory resourcesDirectory, String name) {
@@ -38,7 +38,7 @@ static TransportVersionLatest readLatestFile(Path file) throws IOException {
3838
}
3939

4040
static Directory getDefinitionsDirectory(Directory resourcesDirectory) {
41-
return resourcesDirectory.dir("defined");
41+
return resourcesDirectory.dir("definitions");
4242
}
4343

4444
static Directory getLatestDirectory(Directory resourcesDirectory) {

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ public void validateTransportVersions() throws IOException {
4545
final Predicate<String> referenceChecker;
4646
if (getDefinitionsDirectory().isPresent()) {
4747
Path definitionsDir = getDefinitionsDirectory().getAsFile().get().toPath();
48-
referenceChecker = (name) -> Files.exists(definitionsDir.resolve(name + ".csv"));
48+
referenceChecker = (name) -> Files.exists(definitionsDir.resolve("named/" + name + ".csv"));
4949
} else {
5050
referenceChecker = (name) -> false;
5151
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ public void validateTransportVersions() throws IOException {
9696
return; // no definitions to validate, remove this leniency once all branches have at least one version
9797
}
9898
Path resourcesDir = getResourcesDirectory().getAsFile().get().toPath();
99-
Path definitionsDir = resourcesDir.resolve("defined");
99+
Path definitionsDir = resourcesDir.resolve("definitions");
100100
Path latestDir = resourcesDir.resolve("latest");
101101

102102
// first check which resource files already exist in main
@@ -110,9 +110,11 @@ public void validateTransportVersions() throws IOException {
110110
// now load all definitions, do some validation and record them by various keys for later quick lookup
111111
// NOTE: this must run after loading referenced names and existing definitions
112112
// NOTE: this is sorted so that the order of cross validation is deterministic
113-
try (var definitionsStream = Files.list(definitionsDir).sorted()) {
114-
for (var definitionFile : definitionsStream.toList()) {
115-
recordAndValidateDefinition(readDefinitionFile(definitionFile));
113+
for (String subDir : List.of("initial", "named")) {
114+
try (var definitionsStream = Files.list(definitionsDir.resolve(subDir)).sorted()) {
115+
for (var definitionFile : definitionsStream.toList()) {
116+
recordAndValidateDefinition(readDefinitionFile(definitionFile));
117+
}
116118
}
117119
}
118120

‎server/src/main/java/org/elasticsearch/TransportVersion.java‎

Lines changed: 25 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
import java.util.ArrayList;
2525
import java.util.Collection;
2626
import java.util.Collections;
27-
import java.util.HashMap;
2827
import java.util.List;
2928
import java.util.Map;
3029
import java.util.Objects;
@@ -113,6 +112,7 @@ public static TransportVersion fromBufferedReader(
113112
String component,
114113
String path,
115114
boolean nameInFile,
115+
boolean isNamed,
116116
BufferedReader bufferedReader,
117117
Integer latest
118118
) {
@@ -128,7 +128,14 @@ public static TransportVersion fromBufferedReader(
128128
if (parts.length < (nameInFile ? 2 : 1)) {
129129
throw new IllegalStateException("invalid transport version file format [" + toComponentPath(component, path) + "]");
130130
}
131-
String name = nameInFile ? parts[0] : path.substring(path.lastIndexOf('/') + 1, path.length() - 4);
131+
String name = null;
132+
if (isNamed) {
133+
if (nameInFile) {
134+
name = parts[0];
135+
} else {
136+
name = path.substring(path.lastIndexOf('/') + 1, path.length() - 4);
137+
}
138+
}
132139
List<Integer> ids = new ArrayList<>();
133140
for (int i = nameInFile ? 1 : 0; i < parts.length; ++i) {
134141
try {
@@ -156,7 +163,7 @@ public static TransportVersion fromBufferedReader(
156163
}
157164
}
158165

159-
public static Map<String, TransportVersion> collectFromInputStreams(
166+
public static List<TransportVersion> collectFromInputStreams(
160167
String component,
161168
Function<String, InputStream> nameToStream,
162169
String latestFileName
@@ -165,32 +172,32 @@ public static Map<String, TransportVersion> collectFromInputStreams(
165172
component,
166173
"/transport/latest/" + latestFileName,
167174
nameToStream,
168-
(c, p, br) -> fromBufferedReader(c, p, true, br, Integer.MAX_VALUE)
175+
(c, p, br) -> fromBufferedReader(c, p, true, false, br, Integer.MAX_VALUE)
169176
);
170177
if (latest != null) {
171-
List<String> versionFilesNames = parseFromBufferedReader(
178+
List<String> versionRelativePaths = parseFromBufferedReader(
172179
component,
173-
"/transport/defined/manifest.txt",
180+
"/transport/definitions/manifest.txt",
174181
nameToStream,
175182
(c, p, br) -> br.lines().filter(line -> line.isBlank() == false).toList()
176183
);
177-
if (versionFilesNames != null) {
178-
Map<String, TransportVersion> transportVersions = new HashMap<>();
179-
for (String versionFileName : versionFilesNames) {
184+
if (versionRelativePaths != null) {
185+
List<TransportVersion> transportVersions = new ArrayList<>();
186+
for (String versionRelativePath : versionRelativePaths) {
180187
TransportVersion transportVersion = parseFromBufferedReader(
181188
component,
182-
"/transport/defined/" + versionFileName,
189+
"/transport/definitions/" + versionRelativePath,
183190
nameToStream,
184-
(c, p, br) -> fromBufferedReader(c, p, false, br, latest.id())
191+
(c, p, br) -> fromBufferedReader(c, p, false, versionRelativePath.startsWith("named/"), br, latest.id())
185192
);
186193
if (transportVersion != null) {
187-
transportVersions.put(versionFileName.substring(0, versionFileName.length() - 4), transportVersion);
194+
transportVersions.add(transportVersion);
188195
}
189196
}
190197
return transportVersions;
191198
}
192199
}
193-
return Map.of();
200+
return List.of();
194201
}
195202

196203
private static String toComponentPath(String component, String path) {
@@ -419,12 +426,15 @@ private static class VersionsHolder {
419426
static {
420427
// collect all the transport versions from server and es modules/plugins (defined in server)
421428
List<TransportVersion> allVersions = new ArrayList<>(TransportVersions.DEFINED_VERSIONS);
422-
Map<String, TransportVersion> allVersionsByName = collectFromInputStreams(
429+
List<TransportVersion> streamVersions = collectFromInputStreams(
423430
"<server>",
424431
TransportVersion.class::getResourceAsStream,
425432
Version.CURRENT.major + "." + Version.CURRENT.minor + ".csv"
426433
);
427-
addTransportVersions(allVersionsByName.values(), allVersions).sort(TransportVersion::compareTo);
434+
Map<String, TransportVersion> allVersionsByName = streamVersions.stream()
435+
.filter(tv -> tv.name() != null)
436+
.collect(Collectors.toMap(TransportVersion::name, v -> v));
437+
addTransportVersions(streamVersions, allVersions).sort(TransportVersion::compareTo);
428438

429439
// set version lookup by release before adding serverless versions
430440
// serverless versions should not affect release version

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,6 @@ static TransportVersion def(int id) {
171171
public static final TransportVersion ESQL_SKIP_ES_INDEX_SERIALIZATION = def(8_827_0_00);
172172
public static final TransportVersion ADD_INDEX_BLOCK_TWO_PHASE = def(8_828_0_00);
173173
public static final TransportVersion RESOLVE_CLUSTER_NO_INDEX_EXPRESSION = def(8_829_0_00);
174-
public static final TransportVersion ML_ROLLOVER_LEGACY_INDICES = def(8_830_0_00);
175174
public static final TransportVersion ADD_INCLUDE_FAILURE_INDICES_OPTION = def(8_831_0_00);
176175
public static final TransportVersion ESQL_RESPONSE_PARTIAL = def(8_832_0_00);
177176
public static final TransportVersion RANK_DOC_OPTIONAL_METADATA_FOR_EXPLAIN = def(8_833_0_00);

‎server/src/main/resources/transport/defined/initial_elasticsearch_8_18_5.csv‎ renamed to ‎server/src/main/resources/transport/definitions/initial/initial_elasticsearch_8_18_5.csv‎

File renamed without changes.

0 commit comments

Comments
 (0)