Skip to content

Commit 45b4e8e

Browse files
GalLaloucheGalLalouche
authored
Merge branch 'main' into feature/apis
2 parents f46d1fa + d405d3a commit 45b4e8e

File tree

298 files changed

+13913
-5205
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

298 files changed

+13913
-5205
lines changed

‎.buildkite/hooks/pre-command‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -95,13 +95,11 @@ if [[ "${USE_PROD_DOCKER_CREDENTIALS:-}" == "true" ]]; then
9595
fi
9696

9797
if [[ "${USE_PERF_CREDENTIALS:-}" == "true" ]]; then
98-
PERF_METRICS_HOST=$(vault read -field=es_host /secret/ci/elastic-elasticsearch/esbench-metics)
99-
PERF_METRICS_INDEX="dummy-micro-benchmarks"
100-
PERF_METRICS_USERNAME=$(vault read -field=es_username /secret/ci/elastic-elasticsearch/esbench-metics)
101-
PERF_METRICS_PASSWORD=$(vault read -field=es_password /secret/ci/elastic-elasticsearch/esbench-metics)
98+
PERF_METRICS_HOST=$(vault read -field=es_host /secret/ci/elastic-elasticsearch/microbenchmarks-metrics)
99+
PERF_METRICS_USERNAME=$(vault read -field=es_user /secret/ci/elastic-elasticsearch/microbenchmarks-metrics)
100+
PERF_METRICS_PASSWORD=$(vault read -field=es_password /secret/ci/elastic-elasticsearch/microbenchmarks-metrics)
102101

103102
export PERF_METRICS_HOST
104-
export PERF_METRICS_INDEX
105103
export PERF_METRICS_USERNAME
106104
export PERF_METRICS_PASSWORD
107105
fi

‎.buildkite/scripts/index-micro-benchmark-results.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
jq -c '.[]' "benchmarks/build/result.json" | while read -r doc; do
44
doc=$(echo "$doc" | jq --argjson timestamp "$(date +%s000)" '. + {"@timestamp": $timestamp}')
55
echo "Indexing $(echo "$doc" | jq -r '.benchmark')"
6-
curl -s -X POST "https://$PERF_METRICS_HOST/$PERF_METRICS_INDEX/_doc" \
6+
curl -s -X POST "https://$PERF_METRICS_HOST/metrics-microbenchmarks-default/_doc" \
77
-u "$PERF_METRICS_USERNAME:$PERF_METRICS_PASSWORD" \
88
-H 'Content-Type: application/json' \
99
-d "$doc"

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java‎

Lines changed: 31 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ public class AggregatorBenchmark {
7373
static final int BLOCK_LENGTH = 8 * 1024;
7474
private static final int OP_COUNT = 1024;
7575
private static final int GROUPS = 5;
76+
private static final int TOP_N_LIMIT = 3;
7677

7778
private static final BlockFactory blockFactory = BlockFactory.getInstance(
7879
new NoopCircuitBreaker("noop"),
@@ -90,6 +91,7 @@ public class AggregatorBenchmark {
9091
private static final String TWO_ORDINALS = "two_" + ORDINALS;
9192
private static final String LONGS_AND_BYTES_REFS = LONGS + "_and_" + BYTES_REFS;
9293
private static final String TWO_LONGS_AND_BYTES_REFS = "two_" + LONGS + "_and_" + BYTES_REFS;
94+
private static final String TOP_N_LONGS = "top_n_" + LONGS;
9395

9496
private static final String VECTOR_DOUBLES = "vector_doubles";
9597
private static final String HALF_NULL_DOUBLES = "half_null_doubles";
@@ -147,7 +149,8 @@ static void selfTest() {
147149
TWO_BYTES_REFS,
148150
TWO_ORDINALS,
149151
LONGS_AND_BYTES_REFS,
150-
TWO_LONGS_AND_BYTES_REFS }
152+
TWO_LONGS_AND_BYTES_REFS,
153+
TOP_N_LONGS }
151154
)
152155
public String grouping;
153156

@@ -161,8 +164,7 @@ static void selfTest() {
161164
public String filter;
162165

163166
private static Operator operator(DriverContext driverContext, String grouping, String op, String dataType, String filter) {
164-
165-
if (grouping.equals("none")) {
167+
if (grouping.equals(NONE)) {
166168
return new AggregationOperator(
167169
List.of(supplier(op, dataType, filter).aggregatorFactory(AggregatorMode.SINGLE, List.of(0)).apply(driverContext)),
168170
driverContext
@@ -188,6 +190,9 @@ private static Operator operator(DriverContext driverContext, String grouping, S
188190
new BlockHash.GroupSpec(1, ElementType.LONG),
189191
new BlockHash.GroupSpec(2, ElementType.BYTES_REF)
190192
);
193+
case TOP_N_LONGS -> List.of(
194+
new BlockHash.GroupSpec(0, ElementType.LONG, false, new BlockHash.TopNDef(0, true, true, TOP_N_LIMIT))
195+
);
191196
default -> throw new IllegalArgumentException("unsupported grouping [" + grouping + "]");
192197
};
193198
return new HashAggregationOperator(
@@ -271,10 +276,14 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
271276
case BOOLEANS -> 2;
272277
default -> GROUPS;
273278
};
279+
int availableGroups = switch (grouping) {
280+
case TOP_N_LONGS -> TOP_N_LIMIT;
281+
default -> groups;
282+
};
274283
switch (op) {
275284
case AVG -> {
276285
DoubleBlock dValues = (DoubleBlock) values;
277-
for (int g = 0; g < groups; g++) {
286+
for (int g = 0; g < availableGroups; g++) {
278287
long group = g;
279288
long sum = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).sum();
280289
long count = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).count();
@@ -286,7 +295,7 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
286295
}
287296
case COUNT -> {
288297
LongBlock lValues = (LongBlock) values;
289-
for (int g = 0; g < groups; g++) {
298+
for (int g = 0; g < availableGroups; g++) {
290299
long group = g;
291300
long expected = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).count() * opCount;
292301
if (lValues.getLong(g) != expected) {
@@ -296,7 +305,7 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
296305
}
297306
case COUNT_DISTINCT -> {
298307
LongBlock lValues = (LongBlock) values;
299-
for (int g = 0; g < groups; g++) {
308+
for (int g = 0; g < availableGroups; g++) {
300309
long group = g;
301310
long expected = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).distinct().count();
302311
long count = lValues.getLong(g);
@@ -310,15 +319,15 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
310319
switch (dataType) {
311320
case LONGS -> {
312321
LongBlock lValues = (LongBlock) values;
313-
for (int g = 0; g < groups; g++) {
322+
for (int g = 0; g < availableGroups; g++) {
314323
if (lValues.getLong(g) != (long) g) {
315324
throw new AssertionError(prefix + "expected [" + g + "] but was [" + lValues.getLong(g) + "]");
316325
}
317326
}
318327
}
319328
case DOUBLES -> {
320329
DoubleBlock dValues = (DoubleBlock) values;
321-
for (int g = 0; g < groups; g++) {
330+
for (int g = 0; g < availableGroups; g++) {
322331
if (dValues.getDouble(g) != (long) g) {
323332
throw new AssertionError(prefix + "expected [" + g + "] but was [" + dValues.getDouble(g) + "]");
324333
}
@@ -331,7 +340,7 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
331340
switch (dataType) {
332341
case LONGS -> {
333342
LongBlock lValues = (LongBlock) values;
334-
for (int g = 0; g < groups; g++) {
343+
for (int g = 0; g < availableGroups; g++) {
335344
long group = g;
336345
long expected = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).max().getAsLong();
337346
if (lValues.getLong(g) != expected) {
@@ -341,7 +350,7 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
341350
}
342351
case DOUBLES -> {
343352
DoubleBlock dValues = (DoubleBlock) values;
344-
for (int g = 0; g < groups; g++) {
353+
for (int g = 0; g < availableGroups; g++) {
345354
long group = g;
346355
long expected = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).max().getAsLong();
347356
if (dValues.getDouble(g) != expected) {
@@ -356,7 +365,7 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
356365
switch (dataType) {
357366
case LONGS -> {
358367
LongBlock lValues = (LongBlock) values;
359-
for (int g = 0; g < groups; g++) {
368+
for (int g = 0; g < availableGroups; g++) {
360369
long group = g;
361370
long expected = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).sum() * opCount;
362371
if (lValues.getLong(g) != expected) {
@@ -366,7 +375,7 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
366375
}
367376
case DOUBLES -> {
368377
DoubleBlock dValues = (DoubleBlock) values;
369-
for (int g = 0; g < groups; g++) {
378+
for (int g = 0; g < availableGroups; g++) {
370379
long group = g;
371380
long expected = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).sum() * opCount;
372381
if (dValues.getDouble(g) != expected) {
@@ -391,6 +400,14 @@ private static void checkGroupingBlock(String prefix, String grouping, Block blo
391400
}
392401
}
393402
}
403+
case TOP_N_LONGS -> {
404+
LongBlock groups = (LongBlock) block;
405+
for (int g = 0; g < TOP_N_LIMIT; g++) {
406+
if (groups.getLong(g) != (long) g) {
407+
throw new AssertionError(prefix + "bad group expected [" + g + "] but was [" + groups.getLong(g) + "]");
408+
}
409+
}
410+
}
394411
case INTS -> {
395412
IntBlock groups = (IntBlock) block;
396413
for (int g = 0; g < GROUPS; g++) {
@@ -495,7 +512,7 @@ private static void checkUngrouped(String prefix, String op, String dataType, Pa
495512

496513
private static Page page(BlockFactory blockFactory, String grouping, String blockType) {
497514
Block dataBlock = dataBlock(blockFactory, blockType);
498-
if (grouping.equals("none")) {
515+
if (grouping.equals(NONE)) {
499516
return new Page(dataBlock);
500517
}
501518
List<Block> blocks = groupingBlocks(grouping, blockType);
@@ -564,7 +581,7 @@ private static Block groupingBlock(String grouping, String blockType) {
564581
default -> throw new UnsupportedOperationException("bad grouping [" + grouping + "]");
565582
};
566583
return switch (grouping) {
567-
case LONGS -> {
584+
case TOP_N_LONGS, LONGS -> {
568585
var builder = blockFactory.newLongBlockBuilder(BLOCK_LENGTH);
569586
for (int i = 0; i < BLOCK_LENGTH; i++) {
570587
for (int v = 0; v < valuesPerGroup; v++) {

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoPlugin.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,13 @@
1515
import org.gradle.api.Project;
1616
import org.gradle.api.artifacts.Configuration;
1717
import org.gradle.api.attributes.Category;
18+
import org.gradle.api.attributes.Usage;
1819
import org.gradle.api.plugins.JavaPlugin;
1920

2021
public class DependenciesInfoPlugin implements Plugin<Project> {
22+
23+
public static String USAGE_ATTRIBUTE = "DependenciesInfo";
24+
2125
@Override
2226
public void apply(final Project project) {
2327
project.getPlugins().apply(CompileOnlyResolvePlugin.class);
@@ -43,6 +47,9 @@ public void apply(final Project project) {
4347
)
4448
);
4549

50+
dependenciesInfoFilesConfiguration.attributes(
51+
attributes -> attributes.attribute(Usage.USAGE_ATTRIBUTE, project.getObjects().named(Usage.class, USAGE_ATTRIBUTE))
52+
);
4653
project.getArtifacts().add("dependenciesInfoFiles", depsInfo);
4754

4855
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,11 @@
2424
import java.util.HexFormat;
2525
import java.util.Locale;
2626
import java.util.function.Function;
27+
import java.util.jar.Attributes;
2728
import java.util.jar.JarEntry;
2829
import java.util.jar.JarFile;
2930
import java.util.jar.JarOutputStream;
31+
import java.util.jar.Manifest;
3032
import java.util.stream.Collectors;
3133

3234
import static org.objectweb.asm.ClassWriter.COMPUTE_FRAMES;
@@ -60,6 +62,10 @@ public String toString() {
6062
}
6163
}
6264

65+
public static void patchJar(File inputJar, File outputJar, Collection<PatcherInfo> patchers) {
66+
patchJar(inputJar, outputJar, patchers, false);
67+
}
68+
6369
/**
6470
* Patches the classes in the input JAR file, using the collection of patchers. Each patcher specifies a target class (its jar entry
6571
* name) and the SHA256 digest on the class bytes.
@@ -69,8 +75,11 @@ public String toString() {
6975
* @param inputFile the JAR file to patch
7076
* @param outputFile the output (patched) JAR file
7177
* @param patchers list of patcher info (classes to patch (jar entry name + optional SHA256 digest) and ASM visitor to transform them)
78+
* @param unsignJar whether to remove class signatures from the JAR Manifest; set this to true when patching a signed JAR,
79+
* otherwise the patched classes will fail to load at runtime due to mismatched signatures.
80+
* @see <a href="https://docs.oracle.com/javase/tutorial/deployment/jar/intro.html">Understanding Signing and Verification</a>
7281
*/
73-
public static void patchJar(File inputFile, File outputFile, Collection<PatcherInfo> patchers) {
82+
public static void patchJar(File inputFile, File outputFile, Collection<PatcherInfo> patchers, boolean unsignJar) {
7483
var classPatchers = patchers.stream().collect(Collectors.toMap(PatcherInfo::jarEntryName, Function.identity()));
7584
var mismatchedClasses = new ArrayList<MismatchInfo>();
7685
try (JarFile jarFile = new JarFile(inputFile); JarOutputStream jos = new JarOutputStream(new FileOutputStream(outputFile))) {
@@ -101,9 +110,23 @@ public static void patchJar(File inputFile, File outputFile, Collection<PatcherI
101110
);
102111
}
103112
} else {
104-
// Read the entry's data and write it to the new JAR
105113
try (InputStream is = jarFile.getInputStream(entry)) {
106-
is.transferTo(jos);
114+
if (unsignJar && entryName.equals("META-INF/MANIFEST.MF")) {
115+
var manifest = new Manifest(is);
116+
for (var manifestEntry : manifest.getEntries().entrySet()) {
117+
var nonSignatureAttributes = new Attributes();
118+
for (var attribute : manifestEntry.getValue().entrySet()) {
119+
if (attribute.getKey().toString().endsWith("Digest") == false) {
120+
nonSignatureAttributes.put(attribute.getKey(), attribute.getValue());
121+
}
122+
}
123+
manifestEntry.setValue(nonSignatureAttributes);
124+
}
125+
manifest.write(jos);
126+
} else if (unsignJar == false || entryName.matches("META-INF/.*\\.SF") == false) {
127+
// Read the entry's data and write it to the new JAR
128+
is.transferTo(jos);
129+
}
107130
}
108131
}
109132
jos.closeEntry();

‎distribution/build.gradle‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ configurations {
3131
attributes {
3232
attribute(Category.CATEGORY_ATTRIBUTE, project.getObjects().named(Category.class, Category.DOCUMENTATION))
3333
}
34+
attributes {
35+
attribute(Usage.USAGE_ATTRIBUTE, project.getObjects().named(Usage.class, DependenciesInfoPlugin.USAGE_ATTRIBUTE))
36+
}
3437
}
3538
featuresMetadata {
3639
attributes {

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmErgonomics.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@
2828
*/
2929
final class JvmErgonomics {
3030

31+
static final double DIRECT_MEMORY_TO_HEAP_FACTOR = 0.5;
32+
3133
private JvmErgonomics() {
3234
throw new AssertionError("No instances intended");
3335
}
@@ -44,7 +46,7 @@ static List<String> choose(final List<String> userDefinedJvmOptions, Settings no
4446
final long heapSize = JvmOption.extractMaxHeapSize(finalJvmOptions);
4547
final long maxDirectMemorySize = JvmOption.extractMaxDirectMemorySize(finalJvmOptions);
4648
if (maxDirectMemorySize == 0) {
47-
ergonomicChoices.add("-XX:MaxDirectMemorySize=" + heapSize / 2);
49+
ergonomicChoices.add("-XX:MaxDirectMemorySize=" + (long) (DIRECT_MEMORY_TO_HEAP_FACTOR * heapSize));
4850
}
4951

5052
final boolean tuneG1GCForSmallHeap = tuneG1GCForSmallHeap(heapSize);

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/MachineDependentHeap.java‎

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
import org.elasticsearch.cluster.node.DiscoveryNodeRole;
1313
import org.elasticsearch.common.settings.Settings;
14+
import org.elasticsearch.common.util.FeatureFlag;
1415
import org.elasticsearch.node.NodeRoleSettings;
1516

1617
import java.io.IOException;
@@ -37,6 +38,8 @@ public class MachineDependentHeap {
3738
protected static final long MAX_HEAP_SIZE = GB * 31; // 31GB
3839
protected static final long MIN_HEAP_SIZE = 1024 * 1024 * 128; // 128MB
3940

41+
private static final FeatureFlag NEW_ML_MEMORY_COMPUTATION_FEATURE_FLAG = new FeatureFlag("new_ml_memory_computation");
42+
4043
public MachineDependentHeap() {}
4144

4245
/**
@@ -76,12 +79,16 @@ protected int getHeapSizeMb(Settings nodeSettings, MachineNodeRole role, long av
7679
/*
7780
* Machine learning only node.
7881
*
79-
* <p>Heap is computed as:
80-
* <ul>
81-
* <li>40% of total system memory when total system memory 16 gigabytes or less.</li>
82-
* <li>40% of the first 16 gigabytes plus 10% of memory above that when total system memory is more than 16 gigabytes.</li>
83-
* <li>The absolute maximum heap size is 31 gigabytes.</li>
84-
* </ul>
82+
* The memory reserved for Java is computed as:
83+
* - 40% of total system memory when total system memory 16 gigabytes or less.
84+
* - 40% of the first 16 gigabytes plus 10% of memory above that when total system memory is more than 16 gigabytes.
85+
* - The absolute maximum heap size is 31 gigabytes.
86+
*
87+
* This Java memory is divided as follows:
88+
* - 2/3 of the Java memory is reserved for the Java heap.
89+
* - 1/3 of the Java memory is reserved for the Java direct memory.
90+
*
91+
* The direct memory being half of the heap is set by the JvmErgonomics class.
8592
*
8693
* In all cases the result is rounded down to the next whole multiple of 4 megabytes.
8794
* The reason for doing this is that Java will round requested heap sizes to a multiple
@@ -95,13 +102,22 @@ protected int getHeapSizeMb(Settings nodeSettings, MachineNodeRole role, long av
95102
*
96103
* If this formula is changed then corresponding changes must be made to the {@code NativeMemoryCalculator} and
97104
* {@code MlAutoscalingDeciderServiceTests} classes in the ML plugin code. Failure to keep the logic synchronized
98-
* could result in repeated autoscaling up and down.
105+
* could result in ML processes crashing with OOM errors or repeated autoscaling up and down.
99106
*/
100107
case ML_ONLY -> {
101-
if (availableMemory <= (GB * 16)) {
102-
yield mb((long) (availableMemory * .4), 4);
108+
double heapFractionBelow16GB = 0.4;
109+
double heapFractionAbove16GB = 0.1;
110+
if (NEW_ML_MEMORY_COMPUTATION_FEATURE_FLAG.isEnabled()) {
111+
heapFractionBelow16GB = 0.4 / (1.0 + JvmErgonomics.DIRECT_MEMORY_TO_HEAP_FACTOR);
112+
heapFractionAbove16GB = 0.1 / (1.0 + JvmErgonomics.DIRECT_MEMORY_TO_HEAP_FACTOR);
113+
}
114+
if (availableMemory <= GB * 16) {
115+
yield mb((long) (availableMemory * heapFractionBelow16GB), 4);
103116
} else {
104-
yield mb((long) min((GB * 16) * .4 + (availableMemory - GB * 16) * .1, MAX_HEAP_SIZE), 4);
117+
yield mb(
118+
(long) min(GB * 16 * heapFractionBelow16GB + (availableMemory - GB * 16) * heapFractionAbove16GB, MAX_HEAP_SIZE),
119+
4
120+
);
105121
}
106122
}
107123
/*

0 commit comments

Comments
 (0)