Skip to content

Commit 1096855

Browse files
davidkyledavidkyle
authored
Merge branch 'main' into dont-stop-endpoint
2 parents 2a7a0bd + 4cda8c2 commit 1096855

File tree

305 files changed

+10487
-2440
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

305 files changed

+10487
-2440
lines changed

‎.buildkite/scripts/dra-workflow.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ echo --- Building release artifacts
7171
buildReleaseArtifacts \
7272
exportCompressedDockerImages \
7373
exportDockerContexts \
74+
:zipAggregation \
7475
:distribution:generateDependenciesReport
7576

7677
PATH="$PATH:${JAVA_HOME}/bin" # Required by the following script

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java‎

Lines changed: 31 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ public class AggregatorBenchmark {
7373
static final int BLOCK_LENGTH = 8 * 1024;
7474
private static final int OP_COUNT = 1024;
7575
private static final int GROUPS = 5;
76+
private static final int TOP_N_LIMIT = 3;
7677

7778
private static final BlockFactory blockFactory = BlockFactory.getInstance(
7879
new NoopCircuitBreaker("noop"),
@@ -90,6 +91,7 @@ public class AggregatorBenchmark {
9091
private static final String TWO_ORDINALS = "two_" + ORDINALS;
9192
private static final String LONGS_AND_BYTES_REFS = LONGS + "_and_" + BYTES_REFS;
9293
private static final String TWO_LONGS_AND_BYTES_REFS = "two_" + LONGS + "_and_" + BYTES_REFS;
94+
private static final String TOP_N_LONGS = "top_n_" + LONGS;
9395

9496
private static final String VECTOR_DOUBLES = "vector_doubles";
9597
private static final String HALF_NULL_DOUBLES = "half_null_doubles";
@@ -147,7 +149,8 @@ static void selfTest() {
147149
TWO_BYTES_REFS,
148150
TWO_ORDINALS,
149151
LONGS_AND_BYTES_REFS,
150-
TWO_LONGS_AND_BYTES_REFS }
152+
TWO_LONGS_AND_BYTES_REFS,
153+
TOP_N_LONGS }
151154
)
152155
public String grouping;
153156

@@ -161,8 +164,7 @@ static void selfTest() {
161164
public String filter;
162165

163166
private static Operator operator(DriverContext driverContext, String grouping, String op, String dataType, String filter) {
164-
165-
if (grouping.equals("none")) {
167+
if (grouping.equals(NONE)) {
166168
return new AggregationOperator(
167169
List.of(supplier(op, dataType, filter).aggregatorFactory(AggregatorMode.SINGLE, List.of(0)).apply(driverContext)),
168170
driverContext
@@ -188,6 +190,9 @@ private static Operator operator(DriverContext driverContext, String grouping, S
188190
new BlockHash.GroupSpec(1, ElementType.LONG),
189191
new BlockHash.GroupSpec(2, ElementType.BYTES_REF)
190192
);
193+
case TOP_N_LONGS -> List.of(
194+
new BlockHash.GroupSpec(0, ElementType.LONG, false, new BlockHash.TopNDef(0, true, true, TOP_N_LIMIT))
195+
);
191196
default -> throw new IllegalArgumentException("unsupported grouping [" + grouping + "]");
192197
};
193198
return new HashAggregationOperator(
@@ -271,10 +276,14 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
271276
case BOOLEANS -> 2;
272277
default -> GROUPS;
273278
};
279+
int availableGroups = switch (grouping) {
280+
case TOP_N_LONGS -> TOP_N_LIMIT;
281+
default -> groups;
282+
};
274283
switch (op) {
275284
case AVG -> {
276285
DoubleBlock dValues = (DoubleBlock) values;
277-
for (int g = 0; g < groups; g++) {
286+
for (int g = 0; g < availableGroups; g++) {
278287
long group = g;
279288
long sum = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).sum();
280289
long count = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).count();
@@ -286,7 +295,7 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
286295
}
287296
case COUNT -> {
288297
LongBlock lValues = (LongBlock) values;
289-
for (int g = 0; g < groups; g++) {
298+
for (int g = 0; g < availableGroups; g++) {
290299
long group = g;
291300
long expected = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).count() * opCount;
292301
if (lValues.getLong(g) != expected) {
@@ -296,7 +305,7 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
296305
}
297306
case COUNT_DISTINCT -> {
298307
LongBlock lValues = (LongBlock) values;
299-
for (int g = 0; g < groups; g++) {
308+
for (int g = 0; g < availableGroups; g++) {
300309
long group = g;
301310
long expected = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).distinct().count();
302311
long count = lValues.getLong(g);
@@ -310,15 +319,15 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
310319
switch (dataType) {
311320
case LONGS -> {
312321
LongBlock lValues = (LongBlock) values;
313-
for (int g = 0; g < groups; g++) {
322+
for (int g = 0; g < availableGroups; g++) {
314323
if (lValues.getLong(g) != (long) g) {
315324
throw new AssertionError(prefix + "expected [" + g + "] but was [" + lValues.getLong(g) + "]");
316325
}
317326
}
318327
}
319328
case DOUBLES -> {
320329
DoubleBlock dValues = (DoubleBlock) values;
321-
for (int g = 0; g < groups; g++) {
330+
for (int g = 0; g < availableGroups; g++) {
322331
if (dValues.getDouble(g) != (long) g) {
323332
throw new AssertionError(prefix + "expected [" + g + "] but was [" + dValues.getDouble(g) + "]");
324333
}
@@ -331,7 +340,7 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
331340
switch (dataType) {
332341
case LONGS -> {
333342
LongBlock lValues = (LongBlock) values;
334-
for (int g = 0; g < groups; g++) {
343+
for (int g = 0; g < availableGroups; g++) {
335344
long group = g;
336345
long expected = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).max().getAsLong();
337346
if (lValues.getLong(g) != expected) {
@@ -341,7 +350,7 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
341350
}
342351
case DOUBLES -> {
343352
DoubleBlock dValues = (DoubleBlock) values;
344-
for (int g = 0; g < groups; g++) {
353+
for (int g = 0; g < availableGroups; g++) {
345354
long group = g;
346355
long expected = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).max().getAsLong();
347356
if (dValues.getDouble(g) != expected) {
@@ -356,7 +365,7 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
356365
switch (dataType) {
357366
case LONGS -> {
358367
LongBlock lValues = (LongBlock) values;
359-
for (int g = 0; g < groups; g++) {
368+
for (int g = 0; g < availableGroups; g++) {
360369
long group = g;
361370
long expected = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).sum() * opCount;
362371
if (lValues.getLong(g) != expected) {
@@ -366,7 +375,7 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
366375
}
367376
case DOUBLES -> {
368377
DoubleBlock dValues = (DoubleBlock) values;
369-
for (int g = 0; g < groups; g++) {
378+
for (int g = 0; g < availableGroups; g++) {
370379
long group = g;
371380
long expected = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).sum() * opCount;
372381
if (dValues.getDouble(g) != expected) {
@@ -391,6 +400,14 @@ private static void checkGroupingBlock(String prefix, String grouping, Block blo
391400
}
392401
}
393402
}
403+
case TOP_N_LONGS -> {
404+
LongBlock groups = (LongBlock) block;
405+
for (int g = 0; g < TOP_N_LIMIT; g++) {
406+
if (groups.getLong(g) != (long) g) {
407+
throw new AssertionError(prefix + "bad group expected [" + g + "] but was [" + groups.getLong(g) + "]");
408+
}
409+
}
410+
}
394411
case INTS -> {
395412
IntBlock groups = (IntBlock) block;
396413
for (int g = 0; g < GROUPS; g++) {
@@ -495,7 +512,7 @@ private static void checkUngrouped(String prefix, String op, String dataType, Pa
495512

496513
private static Page page(BlockFactory blockFactory, String grouping, String blockType) {
497514
Block dataBlock = dataBlock(blockFactory, blockType);
498-
if (grouping.equals("none")) {
515+
if (grouping.equals(NONE)) {
499516
return new Page(dataBlock);
500517
}
501518
List<Block> blocks = groupingBlocks(grouping, blockType);
@@ -564,7 +581,7 @@ private static Block groupingBlock(String grouping, String blockType) {
564581
default -> throw new UnsupportedOperationException("bad grouping [" + grouping + "]");
565582
};
566583
return switch (grouping) {
567-
case LONGS -> {
584+
case TOP_N_LONGS, LONGS -> {
568585
var builder = blockFactory.newLongBlockBuilder(BLOCK_LENGTH);
569586
for (int i = 0; i < BLOCK_LENGTH; i++) {
570587
for (int v = 0; v < valuesPerGroup; v++) {

‎build-tools-internal/gradle/wrapper/gradle-wrapper.properties‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
distributionBase=GRADLE_USER_HOME
22
distributionPath=wrapper/dists
3-
distributionSha256Sum=d7042b3c11565c192041fc8c4703f541b888286404b4f267138c1d094d8ecdca
4-
distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.1-all.zip
3+
distributionSha256Sum=443c9c8ee2ac1ee0e11881a40f2376d79c66386264a44b24a9f8ca67e633375f
4+
distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.2-all.zip
55
networkTimeout=10000
66
validateDistributionUrl=true
77
zipStoreBase=GRADLE_USER_HOME

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/MethodReplacement.java‎ renamed to ‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/MethodReplacement.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
* License v3.0 only", or the "Server Side Public License, v 1".
88
*/
99

10-
package org.elasticsearch.gradle.internal.dependencies.patches.hdfs;
10+
package org.elasticsearch.gradle.internal.dependencies.patches;
1111

1212
import org.objectweb.asm.MethodVisitor;
1313
import org.objectweb.asm.Opcodes;
@@ -16,7 +16,7 @@ public class MethodReplacement extends MethodVisitor {
1616
private final MethodVisitor delegate;
1717
private final Runnable bodyWriter;
1818

19-
MethodReplacement(MethodVisitor delegate, Runnable bodyWriter) {
19+
public MethodReplacement(MethodVisitor delegate, Runnable bodyWriter) {
2020
super(Opcodes.ASM9);
2121
this.delegate = delegate;
2222
this.bodyWriter = bodyWriter;

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,11 @@
2424
import java.util.HexFormat;
2525
import java.util.Locale;
2626
import java.util.function.Function;
27+
import java.util.jar.Attributes;
2728
import java.util.jar.JarEntry;
2829
import java.util.jar.JarFile;
2930
import java.util.jar.JarOutputStream;
31+
import java.util.jar.Manifest;
3032
import java.util.stream.Collectors;
3133

3234
import static org.objectweb.asm.ClassWriter.COMPUTE_FRAMES;
@@ -60,6 +62,10 @@ public String toString() {
6062
}
6163
}
6264

65+
public static void patchJar(File inputJar, File outputJar, Collection<PatcherInfo> patchers) {
66+
patchJar(inputJar, outputJar, patchers, false);
67+
}
68+
6369
/**
6470
* Patches the classes in the input JAR file, using the collection of patchers. Each patcher specifies a target class (its jar entry
6571
* name) and the SHA256 digest on the class bytes.
@@ -69,8 +75,11 @@ public String toString() {
6975
* @param inputFile the JAR file to patch
7076
* @param outputFile the output (patched) JAR file
7177
* @param patchers list of patcher info (classes to patch (jar entry name + optional SHA256 digest) and ASM visitor to transform them)
78+
* @param unsignJar whether to remove class signatures from the JAR Manifest; set this to true when patching a signed JAR,
79+
* otherwise the patched classes will fail to load at runtime due to mismatched signatures.
80+
* @see <a href="https://docs.oracle.com/javase/tutorial/deployment/jar/intro.html">Understanding Signing and Verification</a>
7281
*/
73-
public static void patchJar(File inputFile, File outputFile, Collection<PatcherInfo> patchers) {
82+
public static void patchJar(File inputFile, File outputFile, Collection<PatcherInfo> patchers, boolean unsignJar) {
7483
var classPatchers = patchers.stream().collect(Collectors.toMap(PatcherInfo::jarEntryName, Function.identity()));
7584
var mismatchedClasses = new ArrayList<MismatchInfo>();
7685
try (JarFile jarFile = new JarFile(inputFile); JarOutputStream jos = new JarOutputStream(new FileOutputStream(outputFile))) {
@@ -101,9 +110,23 @@ public static void patchJar(File inputFile, File outputFile, Collection<PatcherI
101110
);
102111
}
103112
} else {
104-
// Read the entry's data and write it to the new JAR
105113
try (InputStream is = jarFile.getInputStream(entry)) {
106-
is.transferTo(jos);
114+
if (unsignJar && entryName.equals("META-INF/MANIFEST.MF")) {
115+
var manifest = new Manifest(is);
116+
for (var manifestEntry : manifest.getEntries().entrySet()) {
117+
var nonSignatureAttributes = new Attributes();
118+
for (var attribute : manifestEntry.getValue().entrySet()) {
119+
if (attribute.getKey().toString().endsWith("Digest") == false) {
120+
nonSignatureAttributes.put(attribute.getKey(), attribute.getValue());
121+
}
122+
}
123+
manifestEntry.setValue(nonSignatureAttributes);
124+
}
125+
manifest.write(jos);
126+
} else if (unsignJar == false || entryName.matches("META-INF/.*\\.SF") == false) {
127+
// Read the entry's data and write it to the new JAR
128+
is.transferTo(jos);
129+
}
107130
}
108131
}
109132
jos.closeEntry();

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/azurecore/AzureCoreClassPatcher.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.gradle.internal.dependencies.patches.azurecore;
11+
12+
import org.elasticsearch.gradle.internal.dependencies.patches.PatcherInfo;
13+
import org.elasticsearch.gradle.internal.dependencies.patches.Utils;
14+
import org.gradle.api.artifacts.transform.CacheableTransform;
15+
import org.gradle.api.artifacts.transform.InputArtifact;
16+
import org.gradle.api.artifacts.transform.TransformAction;
17+
import org.gradle.api.artifacts.transform.TransformOutputs;
18+
import org.gradle.api.artifacts.transform.TransformParameters;
19+
import org.gradle.api.file.FileSystemLocation;
20+
import org.gradle.api.provider.Provider;
21+
import org.gradle.api.tasks.Classpath;
22+
import org.jetbrains.annotations.NotNull;
23+
24+
import java.io.File;
25+
import java.util.List;
26+
import java.util.regex.Pattern;
27+
28+
import static org.elasticsearch.gradle.internal.dependencies.patches.PatcherInfo.classPatcher;
29+
30+
@CacheableTransform
31+
public abstract class AzureCoreClassPatcher implements TransformAction<TransformParameters.None> {
32+
33+
private static final String JAR_FILE_TO_PATCH = "azure-core-[\\d.]*\\.jar";
34+
35+
private static final List<PatcherInfo> CLASS_PATCHERS = List.of(
36+
classPatcher(
37+
"com/azure/core/implementation/ImplUtils.class",
38+
"7beda5bdff5ea460cfc08721a188cf07d16e0c987dae45401fca7abf4e6e6c0e",
39+
ImplUtilsPatcher::new
40+
)
41+
);
42+
43+
@Classpath
44+
@InputArtifact
45+
public abstract Provider<FileSystemLocation> getInputArtifact();
46+
47+
@Override
48+
public void transform(@NotNull TransformOutputs outputs) {
49+
File inputFile = getInputArtifact().get().getAsFile();
50+
51+
if (Pattern.matches(JAR_FILE_TO_PATCH, inputFile.getName())) {
52+
System.out.println("Patching " + inputFile.getName());
53+
File outputFile = outputs.file(inputFile.getName().replace(".jar", "-patched.jar"));
54+
Utils.patchJar(inputFile, outputFile, CLASS_PATCHERS, true);
55+
} else {
56+
System.out.println("Skipping " + inputFile.getName());
57+
outputs.file(getInputArtifact());
58+
}
59+
}
60+
61+
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/azurecore/ImplUtilsPatcher.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.gradle.internal.dependencies.patches.azurecore;
11+
12+
import org.elasticsearch.gradle.internal.dependencies.patches.MethodReplacement;
13+
import org.objectweb.asm.ClassVisitor;
14+
import org.objectweb.asm.MethodVisitor;
15+
import org.objectweb.asm.Opcodes;
16+
17+
class ImplUtilsPatcher extends ClassVisitor {
18+
ImplUtilsPatcher(ClassVisitor classVisitor) {
19+
super(Opcodes.ASM9, classVisitor);
20+
}
21+
22+
public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
23+
MethodVisitor mv = super.visitMethod(access, name, descriptor, signature, exceptions);
24+
// `addShutdownHook` invokes `java.lang.Runtime.addShutdownHook`, which is forbidden (i.e. it will throw an Entitlements error).
25+
// We replace the method body here with `return null`.
26+
if (name.equals("addShutdownHookSafely")) {
27+
return new MethodReplacement(mv, () -> {
28+
mv.visitInsn(Opcodes.ACONST_NULL);
29+
mv.visitInsn(Opcodes.ARETURN);
30+
});
31+
}
32+
return mv;
33+
}
34+
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/ShellPatcher.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
package org.elasticsearch.gradle.internal.dependencies.patches.hdfs;
1111

12+
import org.elasticsearch.gradle.internal.dependencies.patches.MethodReplacement;
1213
import org.objectweb.asm.ClassVisitor;
1314
import org.objectweb.asm.ClassWriter;
1415
import org.objectweb.asm.MethodVisitor;

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/ShutdownHookManagerPatcher.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
package org.elasticsearch.gradle.internal.dependencies.patches.hdfs;
1111

12+
import org.elasticsearch.gradle.internal.dependencies.patches.MethodReplacement;
1213
import org.objectweb.asm.ClassVisitor;
1314
import org.objectweb.asm.ClassWriter;
1415
import org.objectweb.asm.MethodVisitor;

‎build-tools-internal/src/main/resources/changelog-schema.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@
8686
"Rollup",
8787
"SQL",
8888
"Search",
89+
"Searchable Snapshots",
8990
"Security",
9091
"Snapshot/Restore",
9192
"Stats",

0 commit comments

Comments
 (0)