Skip to content

Commit f4fbc99

Browse files
ivanceaivancea
committed
Merge branch 'main' into aggs-fix-bucket-script-validation
2 parents 68d1070 + 829fb56 commit f4fbc99

File tree

429 files changed

+8902
-2121
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

429 files changed

+8902
-2121
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ testfixtures_shared/
6969
# Generated
7070
checkstyle_ide.xml
7171
x-pack/plugin/esql/src/main/generated-src/generated/
72+
server/src/main/resources/transport/defined/manifest.txt
7273

7374
# JEnv
7475
.java-version

‎build-tools-internal/build.gradle‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,14 @@ gradlePlugin {
220220
id = 'elasticsearch.internal-yaml-rest-test'
221221
implementationClass = 'org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin'
222222
}
223+
transportVersionManagementPlugin {
224+
id = 'elasticsearch.transport-version-management'
225+
implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin'
226+
}
227+
globalTransportVersionManagementPlugin {
228+
id = 'elasticsearch.global-transport-version-management'
229+
implementationClass = 'org.elasticsearch.gradle.internal.transport.GlobalTransportVersionManagementPlugin'
230+
}
223231
}
224232
}
225233

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import org.elasticsearch.gradle.internal.info.BuildParameterExtension;
1616
import org.elasticsearch.gradle.internal.precommit.JarHellPrecommitPlugin;
1717
import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin;
18+
import org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin;
1819
import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
1920
import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
2021
import org.elasticsearch.gradle.util.GradleUtils;
@@ -36,6 +37,7 @@ public void apply(Project project) {
3637
project.getPluginManager().apply(JarHellPrecommitPlugin.class);
3738
project.getPluginManager().apply(ElasticsearchJavaPlugin.class);
3839
project.getPluginManager().apply(ClusterFeaturesMetadataPlugin.class);
40+
project.getPluginManager().apply(TransportVersionManagementPlugin.class);
3941
boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).getCi();
4042
// Clear default dependencies added by public PluginBuildPlugin as we add our
4143
// own project dependencies for internal builds

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,8 @@ record EarlyAccessJdkBuild(JavaLanguageVersion languageVersion, String buildNumb
6262
@Override
6363
public String url(String os, String arch, String extension) {
6464
// example:
65-
// http://builds.es-jdk-archive.com/jdks/openjdk/26/openjdk-26-ea+6/openjdk-26-ea+6_linux-aarch64_bin.tar.gz
66-
return "http://builds.es-jdk-archive.com/jdks/openjdk/"
65+
// https://builds.es-jdk-archive.com/jdks/openjdk/26/openjdk-26-ea+6/openjdk-26-ea+6_linux-aarch64_bin.tar.gz
66+
return "https://builds.es-jdk-archive.com/jdks/openjdk/"
6767
+ languageVersion.asInt()
6868
+ "/"
6969
+ "openjdk-"

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.gradle.internal.transport;
11+
12+
import org.gradle.api.DefaultTask;
13+
import org.gradle.api.file.ConfigurableFileCollection;
14+
import org.gradle.api.file.RegularFileProperty;
15+
import org.gradle.api.tasks.CacheableTask;
16+
import org.gradle.api.tasks.Classpath;
17+
import org.gradle.api.tasks.OutputFile;
18+
import org.gradle.api.tasks.TaskAction;
19+
import org.objectweb.asm.ClassReader;
20+
import org.objectweb.asm.ClassVisitor;
21+
import org.objectweb.asm.Label;
22+
import org.objectweb.asm.MethodVisitor;
23+
import org.objectweb.asm.Opcodes;
24+
import org.objectweb.asm.tree.LdcInsnNode;
25+
import org.objectweb.asm.tree.MethodNode;
26+
27+
import java.io.IOException;
28+
import java.io.InputStream;
29+
import java.nio.file.FileVisitResult;
30+
import java.nio.file.Files;
31+
import java.nio.file.Path;
32+
import java.nio.file.SimpleFileVisitor;
33+
import java.nio.file.attribute.BasicFileAttributes;
34+
import java.util.HashSet;
35+
import java.util.Set;
36+
37+
/**
38+
* This task locates all method invocations of org.elasticsearch.TransportVersion#fromName(java.lang.String) in the
39+
* provided directory, and then records the value of string literals passed as arguments. It then records each
40+
* string on a newline along with path and line number in the provided output file.
41+
*/
42+
@CacheableTask
43+
public abstract class CollectTransportVersionReferencesTask extends DefaultTask {
44+
public static final String TRANSPORT_VERSION_SET_CLASS = "org/elasticsearch/TransportVersion";
45+
public static final String TRANSPORT_VERSION_SET_METHOD_NAME = "fromName";
46+
public static final String CLASS_EXTENSION = ".class";
47+
public static final String MODULE_INFO = "module-info.class";
48+
49+
/**
50+
* The directory to scan for method invocations.
51+
*/
52+
@Classpath
53+
public abstract ConfigurableFileCollection getClassPath();
54+
55+
/**
56+
* The output file, with each newline containing the string literal argument of each method
57+
* invocation.
58+
*/
59+
@OutputFile
60+
public abstract RegularFileProperty getOutputFile();
61+
62+
@TaskAction
63+
public void checkTransportVersion() throws IOException {
64+
var results = new HashSet<TransportVersionUtils.TransportVersionReference>();
65+
66+
for (var cpElement : getClassPath()) {
67+
Path file = cpElement.toPath();
68+
if (Files.isDirectory(file)) {
69+
addNamesFromClassesDirectory(results, file);
70+
}
71+
}
72+
73+
Path outputFile = getOutputFile().get().getAsFile().toPath();
74+
Files.writeString(outputFile, String.join("\n", results.stream().map(Object::toString).sorted().toList()));
75+
}
76+
77+
private void addNamesFromClassesDirectory(Set<TransportVersionUtils.TransportVersionReference> results, Path file) throws IOException {
78+
Files.walkFileTree(file, new SimpleFileVisitor<>() {
79+
@Override
80+
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
81+
String filename = file.getFileName().toString();
82+
if (filename.endsWith(CLASS_EXTENSION) && filename.endsWith(MODULE_INFO) == false) {
83+
try (var inputStream = Files.newInputStream(file)) {
84+
addNamesFromClass(results, inputStream, classname(file.toString()));
85+
}
86+
}
87+
return FileVisitResult.CONTINUE;
88+
}
89+
});
90+
}
91+
92+
private void addNamesFromClass(Set<TransportVersionUtils.TransportVersionReference> results, InputStream classBytes, String classname)
93+
throws IOException {
94+
ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) {
95+
@Override
96+
public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
97+
return new MethodNode(Opcodes.ASM9, access, name, descriptor, signature, exceptions) {
98+
int lineNumber = -1;
99+
100+
@Override
101+
public void visitLineNumber(int line, Label start) {
102+
lineNumber = line;
103+
}
104+
105+
@Override
106+
public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) {
107+
if (owner.equals(TRANSPORT_VERSION_SET_CLASS) && name.equals(TRANSPORT_VERSION_SET_METHOD_NAME)) {
108+
var abstractInstruction = this.instructions.getLast();
109+
String location = classname + " line " + lineNumber;
110+
if (abstractInstruction instanceof LdcInsnNode ldcInsnNode
111+
&& ldcInsnNode.cst instanceof String tvName
112+
&& tvName.isEmpty() == false) {
113+
results.add(new TransportVersionUtils.TransportVersionReference(tvName, location));
114+
} else {
115+
// The instruction is not a LDC with a String constant (or an empty String), which is not allowed.
116+
throw new RuntimeException(
117+
"TransportVersion.fromName must be called with a non-empty String literal. " + "See " + location + "."
118+
);
119+
}
120+
}
121+
super.visitMethodInsn(opcode, owner, name, descriptor, isInterface);
122+
}
123+
};
124+
}
125+
};
126+
ClassReader classReader = new ClassReader(classBytes);
127+
classReader.accept(classVisitor, 0);
128+
}
129+
130+
private static String classname(String filename) {
131+
return filename.substring(0, filename.length() - CLASS_EXTENSION.length()).replaceAll("[/\\\\]", ".");
132+
}
133+
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.gradle.internal.transport;
11+
12+
import org.gradle.api.DefaultTask;
13+
import org.gradle.api.file.DirectoryProperty;
14+
import org.gradle.api.file.RegularFileProperty;
15+
import org.gradle.api.tasks.InputDirectory;
16+
import org.gradle.api.tasks.OutputFile;
17+
import org.gradle.api.tasks.TaskAction;
18+
19+
import java.io.IOException;
20+
import java.nio.file.Files;
21+
import java.nio.file.Path;
22+
23+
public abstract class GenerateTransportVersionManifestTask extends DefaultTask {
24+
@InputDirectory
25+
public abstract DirectoryProperty getDefinitionsDirectory();
26+
27+
@OutputFile
28+
public abstract RegularFileProperty getManifestFile();
29+
30+
@TaskAction
31+
public void generateTransportVersionManifest() throws IOException {
32+
Path constantsDir = getDefinitionsDirectory().get().getAsFile().toPath();
33+
Path manifestFile = getManifestFile().get().getAsFile().toPath();
34+
try (var writer = Files.newBufferedWriter(manifestFile)) {
35+
try (var stream = Files.list(constantsDir)) {
36+
for (String filename : stream.map(p -> p.getFileName().toString()).toList()) {
37+
if (filename.equals(manifestFile.getFileName().toString())) {
38+
// don't list self
39+
continue;
40+
}
41+
writer.write(filename + "\n");
42+
}
43+
}
44+
}
45+
}
46+
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.gradle.internal.transport;
11+
12+
import org.gradle.api.Plugin;
13+
import org.gradle.api.Project;
14+
import org.gradle.api.artifacts.Configuration;
15+
import org.gradle.api.artifacts.dsl.DependencyHandler;
16+
import org.gradle.api.file.Directory;
17+
import org.gradle.api.plugins.JavaPlugin;
18+
import org.gradle.api.tasks.Copy;
19+
import org.gradle.language.base.plugins.LifecycleBasePlugin;
20+
21+
import java.util.Map;
22+
23+
public class GlobalTransportVersionManagementPlugin implements Plugin<Project> {
24+
25+
@Override
26+
public void apply(Project project) {
27+
project.getPluginManager().apply(LifecycleBasePlugin.class);
28+
29+
DependencyHandler depsHandler = project.getDependencies();
30+
Configuration tvReferencesConfig = project.getConfigurations().create("globalTvReferences");
31+
tvReferencesConfig.setCanBeConsumed(false);
32+
tvReferencesConfig.setCanBeResolved(true);
33+
tvReferencesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute);
34+
35+
// iterate through all projects, and if the management plugin is applied, add that project back as a dep to check
36+
for (Project subProject : project.getRootProject().getSubprojects()) {
37+
subProject.getPlugins().withType(TransportVersionManagementPlugin.class).configureEach(plugin -> {
38+
tvReferencesConfig.getDependencies().add(depsHandler.project(Map.of("path", subProject.getPath())));
39+
});
40+
}
41+
42+
var validateTask = project.getTasks()
43+
.register("validateTransportVersionDefinitions", ValidateTransportVersionDefinitionsTask.class, t -> {
44+
t.setGroup("Transport Versions");
45+
t.setDescription("Validates that all defined TransportVersion constants are used in at least one project");
46+
Directory definitionsDir = TransportVersionUtils.getDefinitionsDirectory(project);
47+
if (definitionsDir.getAsFile().exists()) {
48+
t.getDefinitionsDirectory().set(definitionsDir);
49+
}
50+
t.getReferencesFiles().setFrom(tvReferencesConfig);
51+
});
52+
project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(validateTask));
53+
54+
var generateManifestTask = project.getTasks()
55+
.register("generateTransportVersionManifest", GenerateTransportVersionManifestTask.class, t -> {
56+
t.setGroup("Transport Versions");
57+
t.setDescription("Generate a manifest resource for all the known transport version definitions");
58+
t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project));
59+
t.getManifestFile().set(project.getLayout().getBuildDirectory().file("generated-resources/manifest.txt"));
60+
});
61+
project.getTasks().named(JavaPlugin.PROCESS_RESOURCES_TASK_NAME, Copy.class).configure(t -> {
62+
t.into("transport/defined", c -> c.from(generateManifestTask));
63+
});
64+
}
65+
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.gradle.internal.transport;
11+
12+
import org.elasticsearch.gradle.util.GradleUtils;
13+
import org.gradle.api.Plugin;
14+
import org.gradle.api.Project;
15+
import org.gradle.api.artifacts.Configuration;
16+
import org.gradle.api.file.Directory;
17+
import org.gradle.api.tasks.SourceSet;
18+
import org.gradle.language.base.plugins.LifecycleBasePlugin;
19+
20+
public class TransportVersionManagementPlugin implements Plugin<Project> {
21+
22+
@Override
23+
public void apply(Project project) {
24+
project.getPluginManager().apply(LifecycleBasePlugin.class);
25+
26+
var collectTask = project.getTasks()
27+
.register("collectTransportVersionReferences", CollectTransportVersionReferencesTask.class, t -> {
28+
t.setGroup("Transport Versions");
29+
t.setDescription("Collects all TransportVersion references used throughout the project");
30+
SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME);
31+
t.getClassPath().setFrom(mainSourceSet.getOutput());
32+
t.getOutputFile().set(project.getLayout().getBuildDirectory().file("transport-version/references.txt"));
33+
});
34+
35+
Configuration tvReferencesConfig = project.getConfigurations().create("transportVersionReferences", c -> {
36+
c.setCanBeConsumed(true);
37+
c.setCanBeResolved(false);
38+
c.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute);
39+
});
40+
project.getArtifacts().add(tvReferencesConfig.getName(), collectTask);
41+
42+
var validateTask = project.getTasks()
43+
.register("validateTransportVersionReferences", ValidateTransportVersionReferencesTask.class, t -> {
44+
t.setGroup("Transport Versions");
45+
t.setDescription("Validates that all TransportVersion references used in the project have an associated definition file");
46+
Directory definitionsDir = TransportVersionUtils.getDefinitionsDirectory(project);
47+
if (definitionsDir.getAsFile().exists()) {
48+
t.getDefinitionsDirectory().set(definitionsDir);
49+
}
50+
t.getReferencesFile().set(collectTask.get().getOutputFile());
51+
});
52+
project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(validateTask));
53+
}
54+
}

0 commit comments

Comments
 (0)