Skip to content

Commit 236347e

Browse files
craigtavernercraigtaverner
committed
Merge branch 'agg_values_spatial' of github.com:craigtaverner/elasticsearch into agg_values_spatial
2 parents a4849ea + e66dfeb commit 236347e

File tree

342 files changed

+20662
-5053
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

342 files changed

+20662
-5053
lines changed

‎build-tools-internal/src/main/groovy/elasticsearch.ide.gradle‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -163,8 +163,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
163163
tasks.register('buildDependencyArtifacts') {
164164
group = 'ide'
165165
description = 'Builds artifacts needed as dependency for IDE modules'
166-
dependsOn([':plugins:repository-hdfs:hadoop-client-api:jar',
167-
':x-pack:plugin:esql:compute:ann:jar',
166+
dependsOn([':x-pack:plugin:esql:compute:ann:jar',
168167
':x-pack:plugin:esql:compute:gen:jar',
169168
':server:generateModulesList',
170169
':server:generatePluginsList',

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/HdfsClassPatcher.java‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.gradle.internal.dependencies.patches.hdfs;
11+
12+
import org.gradle.api.artifacts.transform.CacheableTransform;
13+
import org.gradle.api.artifacts.transform.InputArtifact;
14+
import org.gradle.api.artifacts.transform.TransformAction;
15+
import org.gradle.api.artifacts.transform.TransformOutputs;
16+
import org.gradle.api.artifacts.transform.TransformParameters;
17+
import org.gradle.api.file.FileSystemLocation;
18+
import org.gradle.api.provider.Provider;
19+
import org.gradle.api.tasks.Classpath;
20+
import org.gradle.api.tasks.Input;
21+
import org.gradle.api.tasks.Optional;
22+
import org.jetbrains.annotations.NotNull;
23+
import org.objectweb.asm.ClassReader;
24+
import org.objectweb.asm.ClassVisitor;
25+
import org.objectweb.asm.ClassWriter;
26+
27+
import java.io.File;
28+
import java.io.FileOutputStream;
29+
import java.io.IOException;
30+
import java.io.InputStream;
31+
import java.util.Enumeration;
32+
import java.util.HashMap;
33+
import java.util.List;
34+
import java.util.Locale;
35+
import java.util.Map;
36+
import java.util.function.Function;
37+
import java.util.jar.JarEntry;
38+
import java.util.jar.JarFile;
39+
import java.util.jar.JarOutputStream;
40+
import java.util.regex.Pattern;
41+
42+
import static java.util.Map.entry;
43+
import static org.objectweb.asm.ClassWriter.COMPUTE_FRAMES;
44+
import static org.objectweb.asm.ClassWriter.COMPUTE_MAXS;
45+
46+
@CacheableTransform
47+
public abstract class HdfsClassPatcher implements TransformAction<HdfsClassPatcher.Parameters> {
48+
49+
record JarPatchers(String artifactTag, Pattern artifactPattern, Map<String, Function<ClassWriter, ClassVisitor>> jarPatchers) {}
50+
51+
static final List<JarPatchers> allPatchers = List.of(
52+
new JarPatchers(
53+
"hadoop-common",
54+
Pattern.compile("hadoop-common-(?!.*tests)"),
55+
Map.ofEntries(
56+
entry("org/apache/hadoop/util/ShutdownHookManager.class", ShutdownHookManagerPatcher::new),
57+
entry("org/apache/hadoop/util/Shell.class", ShellPatcher::new),
58+
entry("org/apache/hadoop/security/UserGroupInformation.class", SubjectGetSubjectPatcher::new)
59+
)
60+
),
61+
new JarPatchers(
62+
"hadoop-client-api",
63+
Pattern.compile("hadoop-client-api.*"),
64+
Map.ofEntries(
65+
entry("org/apache/hadoop/util/ShutdownHookManager.class", ShutdownHookManagerPatcher::new),
66+
entry("org/apache/hadoop/util/Shell.class", ShellPatcher::new),
67+
entry("org/apache/hadoop/security/UserGroupInformation.class", SubjectGetSubjectPatcher::new),
68+
entry("org/apache/hadoop/security/authentication/client/KerberosAuthenticator.class", SubjectGetSubjectPatcher::new)
69+
)
70+
)
71+
);
72+
73+
interface Parameters extends TransformParameters {
74+
@Input
75+
@Optional
76+
List<String> getMatchingArtifacts();
77+
78+
void setMatchingArtifacts(List<String> matchingArtifacts);
79+
}
80+
81+
@Classpath
82+
@InputArtifact
83+
public abstract Provider<FileSystemLocation> getInputArtifact();
84+
85+
@Override
86+
public void transform(@NotNull TransformOutputs outputs) {
87+
File inputFile = getInputArtifact().get().getAsFile();
88+
89+
List<String> matchingArtifacts = getParameters().getMatchingArtifacts();
90+
List<JarPatchers> patchersToApply = allPatchers.stream()
91+
.filter(jp -> matchingArtifacts.contains(jp.artifactTag()) && jp.artifactPattern().matcher(inputFile.getName()).find())
92+
.toList();
93+
if (patchersToApply.isEmpty()) {
94+
outputs.file(getInputArtifact());
95+
} else {
96+
patchersToApply.forEach(patchers -> {
97+
System.out.println("Patching " + inputFile.getName());
98+
99+
Map<String, Function<ClassWriter, ClassVisitor>> jarPatchers = new HashMap<>(patchers.jarPatchers());
100+
File outputFile = outputs.file(inputFile.getName().replace(".jar", "-patched.jar"));
101+
102+
patchJar(inputFile, outputFile, jarPatchers);
103+
104+
if (jarPatchers.isEmpty() == false) {
105+
throw new IllegalArgumentException(
106+
String.format(
107+
Locale.ROOT,
108+
"error patching [%s] with [%s]: the jar does not contain [%s]",
109+
inputFile.getName(),
110+
patchers.artifactPattern().toString(),
111+
String.join(", ", jarPatchers.keySet())
112+
)
113+
);
114+
}
115+
});
116+
}
117+
}
118+
119+
private static void patchJar(File inputFile, File outputFile, Map<String, Function<ClassWriter, ClassVisitor>> jarPatchers) {
120+
try (JarFile jarFile = new JarFile(inputFile); JarOutputStream jos = new JarOutputStream(new FileOutputStream(outputFile))) {
121+
Enumeration<JarEntry> entries = jarFile.entries();
122+
while (entries.hasMoreElements()) {
123+
JarEntry entry = entries.nextElement();
124+
String entryName = entry.getName();
125+
// Add the entry to the new JAR file
126+
jos.putNextEntry(new JarEntry(entryName));
127+
128+
Function<ClassWriter, ClassVisitor> classPatcher = jarPatchers.remove(entryName);
129+
if (classPatcher != null) {
130+
byte[] classToPatch = jarFile.getInputStream(entry).readAllBytes();
131+
132+
ClassReader classReader = new ClassReader(classToPatch);
133+
ClassWriter classWriter = new ClassWriter(classReader, COMPUTE_FRAMES | COMPUTE_MAXS);
134+
classReader.accept(classPatcher.apply(classWriter), 0);
135+
136+
jos.write(classWriter.toByteArray());
137+
} else {
138+
// Read the entry's data and write it to the new JAR
139+
try (InputStream is = jarFile.getInputStream(entry)) {
140+
is.transferTo(jos);
141+
}
142+
}
143+
jos.closeEntry();
144+
}
145+
} catch (IOException ex) {
146+
throw new RuntimeException(ex);
147+
}
148+
}
149+
}

‎plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/MethodReplacement.java‎ renamed to ‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/MethodReplacement.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
* License v3.0 only", or the "Server Side Public License, v 1".
88
*/
99

10-
package org.elasticsearch.hdfs.patch;
10+
package org.elasticsearch.gradle.internal.dependencies.patches.hdfs;
1111

1212
import org.objectweb.asm.MethodVisitor;
1313
import org.objectweb.asm.Opcodes;

‎plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShellPatcher.java‎ renamed to ‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/ShellPatcher.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
* License v3.0 only", or the "Server Side Public License, v 1".
88
*/
99

10-
package org.elasticsearch.hdfs.patch;
10+
package org.elasticsearch.gradle.internal.dependencies.patches.hdfs;
1111

1212
import org.objectweb.asm.ClassVisitor;
1313
import org.objectweb.asm.ClassWriter;

‎plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShutdownHookManagerPatcher.java‎ renamed to ‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/ShutdownHookManagerPatcher.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
* License v3.0 only", or the "Server Side Public License, v 1".
88
*/
99

10-
package org.elasticsearch.hdfs.patch;
10+
package org.elasticsearch.gradle.internal.dependencies.patches.hdfs;
1111

1212
import org.objectweb.asm.ClassVisitor;
1313
import org.objectweb.asm.ClassWriter;

‎plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/SubjectGetSubjectPatcher.java‎ renamed to ‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/SubjectGetSubjectPatcher.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
* License v3.0 only", or the "Server Side Public License, v 1".
88
*/
99

10-
package org.elasticsearch.hdfs.patch;
10+
package org.elasticsearch.gradle.internal.dependencies.patches.hdfs;
1111

1212
import org.objectweb.asm.ClassVisitor;
1313
import org.objectweb.asm.ClassWriter;

‎distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import org.elasticsearch.core.Tuple;
4949
import org.elasticsearch.env.Environment;
5050
import org.elasticsearch.env.TestEnvironment;
51+
import org.elasticsearch.jdk.RuntimeVersionFeature;
5152
import org.elasticsearch.plugin.scanner.NamedComponentScanner;
5253
import org.elasticsearch.plugins.Platforms;
5354
import org.elasticsearch.plugins.PluginDescriptor;
@@ -891,6 +892,7 @@ public void testInstallMisspelledOfficialPlugins() {
891892
}
892893

893894
public void testBatchFlag() throws Exception {
895+
assumeTrue("security policy validation only available with SecurityManager", RuntimeVersionFeature.isSecurityManagerAvailable());
894896
installPlugin(true);
895897
assertThat(terminal.getErrorOutput(), containsString("WARNING: plugin requires additional permissions"));
896898
assertThat(terminal.getOutput(), containsString("-> Downloading"));
@@ -1529,6 +1531,7 @@ private void assertPolicyConfirmation(Tuple<Path, Environment> pathEnvironmentTu
15291531
}
15301532

15311533
public void testPolicyConfirmation() throws Exception {
1534+
assumeTrue("security policy parsing only available with SecurityManager", RuntimeVersionFeature.isSecurityManagerAvailable());
15321535
writePluginSecurityPolicy(pluginDir, "getClassLoader", "setFactory");
15331536
InstallablePlugin pluginZip = createPluginZip("fake", pluginDir);
15341537

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ private static Stream<String> maybeAttachEntitlementAgent(boolean useEntitlement
180180
}
181181
// We instrument classes in these modules to call the bridge. Because the bridge gets patched
182182
// into java.base, we must export the bridge from java.base to these modules, as a comma-separated list
183-
String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming";
183+
String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
184184
return Stream.of(
185185
"-Des.entitlements.enabled=true",
186186
"-XX:+EnableDynamicAgentLoading",

‎distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,12 @@
1111

1212
import org.elasticsearch.common.settings.Settings;
1313
import org.elasticsearch.common.util.concurrent.EsExecutors;
14+
import org.elasticsearch.core.IOUtils;
1415
import org.elasticsearch.core.Strings;
1516
import org.elasticsearch.test.ESTestCase;
1617
import org.elasticsearch.test.ESTestCase.WithoutSecurityManager;
18+
import org.junit.AfterClass;
19+
import org.junit.BeforeClass;
1720

1821
import java.io.BufferedReader;
1922
import java.io.IOException;
@@ -44,6 +47,19 @@ public class JvmOptionsParserTests extends ESTestCase {
4447

4548
private static final Map<String, String> TEST_SYSPROPS = Map.of("os.name", "Linux", "os.arch", "aarch64");
4649

50+
private static final Path ENTITLEMENTS_LIB_DIR = Path.of("lib", "entitlement-bridge");
51+
52+
@BeforeClass
53+
public static void beforeClass() throws IOException {
54+
Files.createDirectories(ENTITLEMENTS_LIB_DIR);
55+
Files.createTempFile(ENTITLEMENTS_LIB_DIR, "mock-entitlements-bridge", ".jar");
56+
}
57+
58+
@AfterClass
59+
public static void afterClass() throws IOException {
60+
IOUtils.rm(Path.of("lib"));
61+
}
62+
4763
public void testSubstitution() {
4864
final List<String> jvmOptions = JvmOptionsParser.substitutePlaceholders(
4965
List.of("-Djava.io.tmpdir=${ES_TMPDIR}"),

‎docs/changelog/112903.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,5 @@ breaking:
1313
`single-node`.
1414
impact: >-
1515
Remove any value for `discovery.type` from your `elasticsearch.yml`
16-
configuration file.
16+
configuration file except for `multi-node` and `single-node`.
1717
notable: false

0 commit comments

Comments
 (0)