Skip to content

Commit 8c8c7c4

Browse files
rjernstrjernst
committed
iter and add generate manifest
1 parent 1721d2e commit 8c8c7c4

File tree

9 files changed

+177
-91
lines changed

9 files changed

+177
-91
lines changed

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java‎

Lines changed: 85 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -10,35 +10,36 @@
1010
package org.elasticsearch.gradle.internal.transport;
1111

1212
import org.gradle.api.DefaultTask;
13-
import org.gradle.api.file.FileCollection;
13+
import org.gradle.api.file.ConfigurableFileCollection;
1414
import org.gradle.api.file.RegularFileProperty;
15-
import org.gradle.api.provider.Property;
15+
import org.gradle.api.tasks.Classpath;
1616
import org.gradle.api.tasks.InputFiles;
1717
import org.gradle.api.tasks.OutputFile;
1818
import org.gradle.api.tasks.TaskAction;
1919
import org.objectweb.asm.ClassReader;
2020
import org.objectweb.asm.ClassVisitor;
21+
import org.objectweb.asm.Label;
2122
import org.objectweb.asm.MethodVisitor;
2223
import org.objectweb.asm.Opcodes;
2324
import org.objectweb.asm.tree.LdcInsnNode;
2425
import org.objectweb.asm.tree.MethodNode;
2526

26-
import java.io.File;
27-
import java.io.FileInputStream;
28-
import java.io.FileWriter;
2927
import java.io.IOException;
3028
import java.io.InputStream;
31-
import java.util.ArrayList;
32-
import java.util.Arrays;
33-
import java.util.Collection;
29+
import java.nio.file.FileVisitResult;
30+
import java.nio.file.Files;
31+
import java.nio.file.Path;
32+
import java.nio.file.SimpleFileVisitor;
33+
import java.nio.file.attribute.BasicFileAttributes;
3434
import java.util.HashSet;
35-
import java.util.List;
3635
import java.util.Set;
36+
import java.util.jar.JarInputStream;
37+
import java.util.zip.ZipEntry;
3738

3839
/**
3940
* This task locates all method invocations of org.elasticsearch.TransportVersion#fromName(java.lang.String) in the
4041
* provided directory, and then records the value of string literals passed as arguments. It then records each
41-
* string on a newline in the provided output file.
42+
* string on a newline along with path and line number in the provided output file.
4243
*/
4344
public abstract class CollectTransportVersionNamesTask extends DefaultTask {
4445
public static final String TRANSPORT_VERSION_SET_CLASS = "org/elasticsearch/TransportVersion";
@@ -50,7 +51,8 @@ public abstract class CollectTransportVersionNamesTask extends DefaultTask {
5051
* The directory to scan for method invocations.
5152
*/
5253
@InputFiles
53-
public abstract Property<FileCollection> getClassDirs();
54+
@Classpath
55+
public abstract ConfigurableFileCollection getClassPath();
5456

5557
/**
5658
* The output file, with each newline containing the string literal argument of each method
@@ -60,72 +62,89 @@ public abstract class CollectTransportVersionNamesTask extends DefaultTask {
6062
public abstract RegularFileProperty getOutputFile();
6163

6264
@TaskAction
63-
public void checkTransportVersion() {
64-
var classFiles = findJavaClassFiles(getClassDirs().get().getFiles());
65-
var tvNames = getTVDeclarationNames(classFiles);
65+
public void checkTransportVersion() throws IOException {
66+
var results = new HashSet<TransportVersionUtils.TransportVersionReference>();
6667

67-
File file = getOutputFile().get().getAsFile();
68-
try (FileWriter writer = new FileWriter(file)) {
69-
for (String tvName : tvNames) {
70-
writer.write(tvName + "\n");
68+
for (var cpElement : getClassPath()) {
69+
Path file = cpElement.toPath();
70+
if (Files.isDirectory(file)) {
71+
addNamesFromClassesDirectory(results, file);
72+
} else {
73+
assert file.getFileName().toString().endsWith(".jar");
74+
addNamesFromJar(results, file);
7175
}
72-
} catch (IOException e) {
73-
throw new RuntimeException(e);
7476
}
77+
78+
Path outputFile = getOutputFile().get().getAsFile().toPath();
79+
Files.writeString(outputFile, String.join("\n", results.stream().map(Object::toString).sorted().toList()));
7580
}
7681

77-
public static Set<String> getTVDeclarationNames(Collection<File> classfiles) {
78-
var results = new HashSet<String>();
79-
for (File javaFile : classfiles) {
80-
try (InputStream inputStream = new FileInputStream(javaFile)) {
81-
ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) {
82-
@Override
83-
public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
84-
return new MethodNode(Opcodes.ASM9, access, name, descriptor, signature, exceptions) {
85-
@Override
86-
public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) {
87-
if (owner.equals(TRANSPORT_VERSION_SET_CLASS) && name.equals(TRANSPORT_VERSION_SET_METHOD_NAME)) {
88-
var abstractInstruction = this.instructions.getLast();
89-
if (abstractInstruction instanceof LdcInsnNode ldcInsnNode
90-
&& ldcInsnNode.cst instanceof String tvName
91-
&& tvName.isEmpty() == false) {
92-
results.add(tvName);
93-
} else {
94-
// The instruction is not a LDC with a String constant (or an empty String),
95-
// which is not allowed.
96-
throw new RuntimeException(
97-
"Transport Versions must be declared with a constant and non-empty String. "
98-
+ "file: "
99-
+ javaFile.getPath()
100-
);
101-
}
102-
}
103-
super.visitMethodInsn(opcode, owner, name, descriptor, isInterface);
104-
}
105-
};
82+
private void addNamesFromClassesDirectory(Set<TransportVersionUtils.TransportVersionReference> results, Path file) throws IOException {
83+
Files.walkFileTree(file, new SimpleFileVisitor<>() {
84+
@Override
85+
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
86+
String filename = file.getFileName().toString();
87+
if (filename.endsWith(CLASS_EXTENSION) && filename.endsWith(MODULE_INFO) == false) {
88+
try (var inputStream = Files.newInputStream(file)) {
89+
addNamesFromClass(results, inputStream, classname(file.toString()));
10690
}
107-
};
108-
ClassReader classReader = new ClassReader(inputStream);
109-
classReader.accept(classVisitor, 0);
110-
} catch (IOException e) {
111-
throw new RuntimeException(e);
91+
}
92+
return FileVisitResult.CONTINUE;
11293
}
113-
}
114-
return results;
94+
});
11595
}
11696

117-
private static List<File> findJavaClassFiles(Collection<File> files) {
118-
List<File> classFiles = new ArrayList<>();
119-
for (File file : files) {
120-
if (file.isDirectory()) {
121-
File[] subFiles = file.listFiles();
122-
if (subFiles != null) {
123-
classFiles.addAll(findJavaClassFiles(Arrays.asList(subFiles)));
97+
private void addNamesFromJar(Set<TransportVersionUtils.TransportVersionReference> results, Path file) throws IOException {
98+
try (var jar = new JarInputStream(Files.newInputStream(file))) {
99+
ZipEntry entry;
100+
while ((entry = jar.getNextEntry()) != null) {
101+
String filename = entry.getName();
102+
if (filename.endsWith(CLASS_EXTENSION) && filename.endsWith(MODULE_INFO) == false) {
103+
addNamesFromClass(results, jar, classname(entry.toString()));
124104
}
125-
} else if (file.getName().endsWith(CLASS_EXTENSION) && file.getName().endsWith(MODULE_INFO) == false) {
126-
classFiles.add(file);
127105
}
128106
}
129-
return classFiles;
107+
}
108+
109+
private void addNamesFromClass(Set<TransportVersionUtils.TransportVersionReference> results, InputStream classBytes, String classname)
110+
throws IOException {
111+
ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) {
112+
@Override
113+
public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
114+
return new MethodNode(Opcodes.ASM9, access, name, descriptor, signature, exceptions) {
115+
int lineNumber = -1;
116+
117+
@Override
118+
public void visitLineNumber(int line, Label start) {
119+
lineNumber = line;
120+
}
121+
122+
@Override
123+
public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) {
124+
if (owner.equals(TRANSPORT_VERSION_SET_CLASS) && name.equals(TRANSPORT_VERSION_SET_METHOD_NAME)) {
125+
var abstractInstruction = this.instructions.getLast();
126+
String location = classname + " line " + lineNumber;
127+
if (abstractInstruction instanceof LdcInsnNode ldcInsnNode
128+
&& ldcInsnNode.cst instanceof String tvName
129+
&& tvName.isEmpty() == false) {
130+
results.add(new TransportVersionUtils.TransportVersionReference(tvName, location));
131+
} else {
132+
// The instruction is not a LDC with a String constant (or an empty String), which is not allowed.
133+
throw new RuntimeException(
134+
"TransportVersion.fromName must be called with a non-empty String literal. " + "See " + location + "."
135+
);
136+
}
137+
}
138+
super.visitMethodInsn(opcode, owner, name, descriptor, isInterface);
139+
}
140+
};
141+
}
142+
};
143+
ClassReader classReader = new ClassReader(classBytes);
144+
classReader.accept(classVisitor, 0);
145+
}
146+
147+
private static String classname(String filename) {
148+
return filename.substring(0, filename.length() - CLASS_EXTENSION.length()).replace('/', '.');
130149
}
131150
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.gradle.internal.transport;
11+
12+
import org.gradle.api.DefaultTask;
13+
import org.gradle.api.file.DirectoryProperty;
14+
import org.gradle.api.file.RegularFileProperty;
15+
import org.gradle.api.tasks.InputDirectory;
16+
import org.gradle.api.tasks.OutputFile;
17+
import org.gradle.api.tasks.TaskAction;
18+
19+
import java.io.IOException;
20+
import java.nio.file.Files;
21+
import java.nio.file.Path;
22+
23+
public abstract class GenerateTransportVersionManifestTask extends DefaultTask {
24+
@InputDirectory
25+
public abstract DirectoryProperty getConstantsDirectory();
26+
27+
@OutputFile
28+
public abstract RegularFileProperty getManifestFile();
29+
30+
@TaskAction
31+
public void generateTransportVersionManifest() throws IOException {
32+
Path constantsDir = getConstantsDirectory().get().getAsFile().toPath();
33+
Path manifestFile = getManifestFile().get().getAsFile().toPath();
34+
try (var writer = Files.newBufferedWriter(manifestFile)) {
35+
try (var stream = Files.list(constantsDir)) {
36+
for (String filename : stream.map(p -> p.getFileName().toString()).toList()) {
37+
writer.write(filename + "\n");
38+
}
39+
}
40+
}
41+
}
42+
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,14 +39,14 @@ public void apply(Project project) {
3939
tvDependencies.add(depsHandler.project(Map.of("path", ":server")));
4040

4141
Configuration tvNamesConfig = project.getConfigurations().detachedConfiguration(tvDependencies.toArray(new Dependency[0]));
42-
tvNamesConfig.attributes(TransportVersionUtils::addTransportVersionNamesAttribute);
42+
tvNamesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute);
4343

4444
var validateTask = project.getTasks()
4545
.register("validateTransportVersionConstants", ValidateTransportVersionConstantsTask.class, t -> {
4646
t.setGroup("Transport Versions");
4747
t.setDescription("Validates that all defined TransportVersion constants are used in at least one project");
4848
t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project));
49-
t.getNamesFiles().setFrom(tvNamesConfig);
49+
t.getReferencesFiles().setFrom(tvNamesConfig);
5050
});
5151

5252
project.getTasks().named("check").configure(t -> t.dependsOn(validateTask));

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,14 @@ public void apply(Project project) {
2424
t.setGroup("Transport Versions");
2525
t.setDescription("Collects all TransportVersion names used throughout the project");
2626
SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME);
27-
t.getClassDirs().set(mainSourceSet.getRuntimeClasspath());
27+
t.getClassPath().setFrom(mainSourceSet.getRuntimeClasspath());
2828
t.getOutputFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile));
2929
});
3030

3131
Configuration transportVersionsConfig = project.getConfigurations().create("transportVersionNames", c -> {
3232
c.setCanBeConsumed(true);
3333
c.setCanBeResolved(false);
34-
c.attributes(TransportVersionUtils::addTransportVersionNamesAttribute);
34+
c.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute);
3535
});
3636

3737
project.getArtifacts().add(transportVersionsConfig.getName(), collectTask);
@@ -41,7 +41,7 @@ public void apply(Project project) {
4141
t.setGroup("Transport Versions");
4242
t.setDescription("Validates that all TransportVersion names used in the project have an associated data file");
4343
t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project));
44-
t.getNamesFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile));
44+
t.getReferencesFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile));
4545
t.dependsOn(collectTask);
4646

4747
});

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java‎

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,18 @@
2828

2929
class TransportVersionUtils {
3030

31-
static final Attribute<Boolean> TRANSPORT_VERSION_NAMES_ATTRIBUTE = Attribute.of("transport-version-names", Boolean.class);
31+
static final Attribute<Boolean> TRANSPORT_VERSION_REFERENCES_ATTRIBUTE = Attribute.of("transport-version-references", Boolean.class);
3232

33-
record TransportVersionData(String name, List<Integer> ids) {}
33+
record TransportVersionConstant(String name, List<Integer> ids) {}
3434

35-
static TransportVersionData readDataFile(Path file) throws IOException {
35+
record TransportVersionReference(String name, String location) {
36+
@Override
37+
public String toString() {
38+
return name + " " + location;
39+
}
40+
}
41+
42+
static TransportVersionConstant readConstantFile(Path file) throws IOException {
3643
assert file.endsWith(".csv");
3744
String rawName = file.getFileName().toString();
3845
String name = rawName.substring(0, rawName.length() - 4);
@@ -49,16 +56,30 @@ static TransportVersionData readDataFile(Path file) throws IOException {
4956
if (Comparators.isInOrder(ids, Comparator.reverseOrder()) == false) {
5057
throw new IOException("invalid transport version data file [" + file + "], ids are not in sorted");
5158
}
52-
return new TransportVersionData(name, ids);
59+
return new TransportVersionConstant(name, ids);
60+
}
61+
62+
static List<TransportVersionReference> readReferencesFile(Path file) throws IOException {
63+
assert file.endsWith(".txt");
64+
List<TransportVersionReference> results = new ArrayList<>();
65+
for (String line : Files.readAllLines(file, StandardCharsets.UTF_8)) {
66+
String[] parts = line.split(" ", 2);
67+
if (parts.length != 2) {
68+
throw new IOException("Invalid transport version data file [" + file + "]: " + line);
69+
}
70+
results.add(new TransportVersionReference(parts[0], parts[1]));
71+
}
72+
return results;
5373
}
5474

5575
static Directory getConstantsDirectory(Project project) {
5676
Directory serverDir = project.getRootProject().project(":server").getLayout().getProjectDirectory();
5777
return serverDir.dir("src/main/resources/transport/constants");
5878
}
5979

60-
static void addTransportVersionNamesAttribute(AttributeContainer attributes) {
61-
attributes.attribute(ARTIFACT_TYPE_ATTRIBUTE, "csv");
62-
attributes.attribute(TransportVersionUtils.TRANSPORT_VERSION_NAMES_ATTRIBUTE, true);
80+
static void addTransportVersionReferencesAttribute(AttributeContainer attributes) {
81+
attributes.attribute(ARTIFACT_TYPE_ATTRIBUTE, "txt");
82+
attributes.attribute(TransportVersionUtils.TRANSPORT_VERSION_REFERENCES_ATTRIBUTE, true);
6383
}
84+
6485
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
package org.elasticsearch.gradle.internal.transport;
1111

12+
import org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TransportVersionReference;
1213
import org.gradle.api.DefaultTask;
1314
import org.gradle.api.file.ConfigurableFileCollection;
1415
import org.gradle.api.file.DirectoryProperty;
@@ -17,12 +18,14 @@
1718
import org.gradle.api.tasks.TaskAction;
1819

1920
import java.io.IOException;
20-
import java.nio.charset.StandardCharsets;
2121
import java.nio.file.Files;
2222
import java.nio.file.Path;
2323
import java.util.HashSet;
2424
import java.util.Set;
2525

26+
import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readConstantFile;
27+
import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readReferencesFile;
28+
2629
/**
2730
* Validates that each defined transport version constant is referenced by at least one project.
2831
*/
@@ -32,20 +35,20 @@ public abstract class ValidateTransportVersionConstantsTask extends DefaultTask
3235
public abstract DirectoryProperty getConstantsDirectory();
3336

3437
@InputFiles
35-
public abstract ConfigurableFileCollection getNamesFiles();
38+
public abstract ConfigurableFileCollection getReferencesFiles();
3639

3740
@TaskAction
3841
public void validateTransportVersions() throws IOException {
3942
Path constantsDir = getConstantsDirectory().getAsFile().get().toPath();
4043

4144
Set<String> allTvNames = new HashSet<>();
42-
for (var tvNamesFile : getNamesFiles()) {
43-
allTvNames.addAll(Files.readAllLines(tvNamesFile.toPath(), StandardCharsets.UTF_8));
45+
for (var tvReferencesFile : getReferencesFiles()) {
46+
readReferencesFile(tvReferencesFile.toPath()).stream().map(TransportVersionReference::name).forEach(allTvNames::add);
4447
}
4548

4649
try (var constantsStream = Files.list(constantsDir)) {
4750
for (var constantsFile : constantsStream.toList()) {
48-
var tv = TransportVersionUtils.readDataFile(constantsFile);
51+
var tv = readConstantFile(constantsFile);
4952
if (allTvNames.contains(tv.name()) == false) {
5053
throw new IllegalStateException("Transport version constant " + tv.name() + " is not referenced");
5154
}

0 commit comments

Comments
 (0)