Skip to content

Commit 9e5b934

Browse files
ldematteldematte
authored
Merge branch 'main' into bc-upgrade/bc-commit-hash-script
2 parents f7f2b07 + a47e00f commit 9e5b934

File tree

661 files changed

+21879
-7097
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

661 files changed

+21879
-7097
lines changed

‎.buildkite/hooks/pre-command‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -95,13 +95,11 @@ if [[ "${USE_PROD_DOCKER_CREDENTIALS:-}" == "true" ]]; then
9595
fi
9696

9797
if [[ "${USE_PERF_CREDENTIALS:-}" == "true" ]]; then
98-
PERF_METRICS_HOST=$(vault read -field=es_host /secret/ci/elastic-elasticsearch/esbench-metics)
99-
PERF_METRICS_INDEX="dummy-micro-benchmarks"
100-
PERF_METRICS_USERNAME=$(vault read -field=es_username /secret/ci/elastic-elasticsearch/esbench-metics)
101-
PERF_METRICS_PASSWORD=$(vault read -field=es_password /secret/ci/elastic-elasticsearch/esbench-metics)
98+
PERF_METRICS_HOST=$(vault read -field=es_host /secret/ci/elastic-elasticsearch/microbenchmarks-metrics)
99+
PERF_METRICS_USERNAME=$(vault read -field=es_user /secret/ci/elastic-elasticsearch/microbenchmarks-metrics)
100+
PERF_METRICS_PASSWORD=$(vault read -field=es_password /secret/ci/elastic-elasticsearch/microbenchmarks-metrics)
102101

103102
export PERF_METRICS_HOST
104-
export PERF_METRICS_INDEX
105103
export PERF_METRICS_USERNAME
106104
export PERF_METRICS_PASSWORD
107105
fi

‎.buildkite/scripts/dra-workflow.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ echo --- Building release artifacts
7171
buildReleaseArtifacts \
7272
exportCompressedDockerImages \
7373
exportDockerContexts \
74+
:zipAggregation \
7475
:distribution:generateDependenciesReport
7576

7677
PATH="$PATH:${JAVA_HOME}/bin" # Required by the following script

‎.buildkite/scripts/index-micro-benchmark-results.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
jq -c '.[]' "benchmarks/build/result.json" | while read -r doc; do
44
doc=$(echo "$doc" | jq --argjson timestamp "$(date +%s000)" '. + {"@timestamp": $timestamp}')
55
echo "Indexing $(echo "$doc" | jq -r '.benchmark')"
6-
curl -s -X POST "https://$PERF_METRICS_HOST/$PERF_METRICS_INDEX/_doc" \
6+
curl -s -X POST "https://$PERF_METRICS_HOST/metrics-microbenchmarks-default/_doc" \
77
-u "$PERF_METRICS_USERNAME:$PERF_METRICS_PASSWORD" \
88
-H 'Content-Type: application/json' \
99
-d "$doc"

‎CONTRIBUTING.md‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -168,16 +168,13 @@ You can import the Elasticsearch project into IntelliJ IDEA via:
168168

169169
#### Checkstyle
170170

171-
If you have the [Checkstyle] plugin installed, you can configure IntelliJ to
172-
check the Elasticsearch code. However, the Checkstyle configuration file does
173-
not work by default with the IntelliJ plugin, so instead an IDE-specific config
174-
file is generated automatically after IntelliJ finishes syncing. You can
175-
manually generate the file with `./gradlew configureIdeCheckstyle` in case
176-
it is removed due to a `./gradlew clean` or other action.
177-
178-
IntelliJ should be automatically configured to use the generated rules after
179-
import via the `.idea/checkstyle-idea.xml` configuration file. No further
180-
action is required.
171+
IntelliJ should automatically configure checkstyle. It does so by running
172+
`configureIdeCheckstyle` on import. That makes `.idea/checkstyle-idea.xml`
173+
configuration file. IntelliJ points checkstyle at that.
174+
175+
Things like `./gradlew clean` or `git clean -xdf` can nuke the file. You can
176+
regenerate it by running `./gradlew -Didea.active=true configureIdeCheckstyle`,
177+
but generally shouldn't have to.
181178

182179
#### Formatting
183180

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java‎

Lines changed: 31 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ public class AggregatorBenchmark {
7373
static final int BLOCK_LENGTH = 8 * 1024;
7474
private static final int OP_COUNT = 1024;
7575
private static final int GROUPS = 5;
76+
private static final int TOP_N_LIMIT = 3;
7677

7778
private static final BlockFactory blockFactory = BlockFactory.getInstance(
7879
new NoopCircuitBreaker("noop"),
@@ -90,6 +91,7 @@ public class AggregatorBenchmark {
9091
private static final String TWO_ORDINALS = "two_" + ORDINALS;
9192
private static final String LONGS_AND_BYTES_REFS = LONGS + "_and_" + BYTES_REFS;
9293
private static final String TWO_LONGS_AND_BYTES_REFS = "two_" + LONGS + "_and_" + BYTES_REFS;
94+
private static final String TOP_N_LONGS = "top_n_" + LONGS;
9395

9496
private static final String VECTOR_DOUBLES = "vector_doubles";
9597
private static final String HALF_NULL_DOUBLES = "half_null_doubles";
@@ -147,7 +149,8 @@ static void selfTest() {
147149
TWO_BYTES_REFS,
148150
TWO_ORDINALS,
149151
LONGS_AND_BYTES_REFS,
150-
TWO_LONGS_AND_BYTES_REFS }
152+
TWO_LONGS_AND_BYTES_REFS,
153+
TOP_N_LONGS }
151154
)
152155
public String grouping;
153156

@@ -161,8 +164,7 @@ static void selfTest() {
161164
public String filter;
162165

163166
private static Operator operator(DriverContext driverContext, String grouping, String op, String dataType, String filter) {
164-
165-
if (grouping.equals("none")) {
167+
if (grouping.equals(NONE)) {
166168
return new AggregationOperator(
167169
List.of(supplier(op, dataType, filter).aggregatorFactory(AggregatorMode.SINGLE, List.of(0)).apply(driverContext)),
168170
driverContext
@@ -188,6 +190,9 @@ private static Operator operator(DriverContext driverContext, String grouping, S
188190
new BlockHash.GroupSpec(1, ElementType.LONG),
189191
new BlockHash.GroupSpec(2, ElementType.BYTES_REF)
190192
);
193+
case TOP_N_LONGS -> List.of(
194+
new BlockHash.GroupSpec(0, ElementType.LONG, false, new BlockHash.TopNDef(0, true, true, TOP_N_LIMIT))
195+
);
191196
default -> throw new IllegalArgumentException("unsupported grouping [" + grouping + "]");
192197
};
193198
return new HashAggregationOperator(
@@ -271,10 +276,14 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
271276
case BOOLEANS -> 2;
272277
default -> GROUPS;
273278
};
279+
int availableGroups = switch (grouping) {
280+
case TOP_N_LONGS -> TOP_N_LIMIT;
281+
default -> groups;
282+
};
274283
switch (op) {
275284
case AVG -> {
276285
DoubleBlock dValues = (DoubleBlock) values;
277-
for (int g = 0; g < groups; g++) {
286+
for (int g = 0; g < availableGroups; g++) {
278287
long group = g;
279288
long sum = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).sum();
280289
long count = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).count();
@@ -286,7 +295,7 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
286295
}
287296
case COUNT -> {
288297
LongBlock lValues = (LongBlock) values;
289-
for (int g = 0; g < groups; g++) {
298+
for (int g = 0; g < availableGroups; g++) {
290299
long group = g;
291300
long expected = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).count() * opCount;
292301
if (lValues.getLong(g) != expected) {
@@ -296,7 +305,7 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
296305
}
297306
case COUNT_DISTINCT -> {
298307
LongBlock lValues = (LongBlock) values;
299-
for (int g = 0; g < groups; g++) {
308+
for (int g = 0; g < availableGroups; g++) {
300309
long group = g;
301310
long expected = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).distinct().count();
302311
long count = lValues.getLong(g);
@@ -310,15 +319,15 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
310319
switch (dataType) {
311320
case LONGS -> {
312321
LongBlock lValues = (LongBlock) values;
313-
for (int g = 0; g < groups; g++) {
322+
for (int g = 0; g < availableGroups; g++) {
314323
if (lValues.getLong(g) != (long) g) {
315324
throw new AssertionError(prefix + "expected [" + g + "] but was [" + lValues.getLong(g) + "]");
316325
}
317326
}
318327
}
319328
case DOUBLES -> {
320329
DoubleBlock dValues = (DoubleBlock) values;
321-
for (int g = 0; g < groups; g++) {
330+
for (int g = 0; g < availableGroups; g++) {
322331
if (dValues.getDouble(g) != (long) g) {
323332
throw new AssertionError(prefix + "expected [" + g + "] but was [" + dValues.getDouble(g) + "]");
324333
}
@@ -331,7 +340,7 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
331340
switch (dataType) {
332341
case LONGS -> {
333342
LongBlock lValues = (LongBlock) values;
334-
for (int g = 0; g < groups; g++) {
343+
for (int g = 0; g < availableGroups; g++) {
335344
long group = g;
336345
long expected = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).max().getAsLong();
337346
if (lValues.getLong(g) != expected) {
@@ -341,7 +350,7 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
341350
}
342351
case DOUBLES -> {
343352
DoubleBlock dValues = (DoubleBlock) values;
344-
for (int g = 0; g < groups; g++) {
353+
for (int g = 0; g < availableGroups; g++) {
345354
long group = g;
346355
long expected = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).max().getAsLong();
347356
if (dValues.getDouble(g) != expected) {
@@ -356,7 +365,7 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
356365
switch (dataType) {
357366
case LONGS -> {
358367
LongBlock lValues = (LongBlock) values;
359-
for (int g = 0; g < groups; g++) {
368+
for (int g = 0; g < availableGroups; g++) {
360369
long group = g;
361370
long expected = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).sum() * opCount;
362371
if (lValues.getLong(g) != expected) {
@@ -366,7 +375,7 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
366375
}
367376
case DOUBLES -> {
368377
DoubleBlock dValues = (DoubleBlock) values;
369-
for (int g = 0; g < groups; g++) {
378+
for (int g = 0; g < availableGroups; g++) {
370379
long group = g;
371380
long expected = LongStream.range(0, BLOCK_LENGTH).filter(l -> l % groups == group).sum() * opCount;
372381
if (dValues.getDouble(g) != expected) {
@@ -391,6 +400,14 @@ private static void checkGroupingBlock(String prefix, String grouping, Block blo
391400
}
392401
}
393402
}
403+
case TOP_N_LONGS -> {
404+
LongBlock groups = (LongBlock) block;
405+
for (int g = 0; g < TOP_N_LIMIT; g++) {
406+
if (groups.getLong(g) != (long) g) {
407+
throw new AssertionError(prefix + "bad group expected [" + g + "] but was [" + groups.getLong(g) + "]");
408+
}
409+
}
410+
}
394411
case INTS -> {
395412
IntBlock groups = (IntBlock) block;
396413
for (int g = 0; g < GROUPS; g++) {
@@ -495,7 +512,7 @@ private static void checkUngrouped(String prefix, String op, String dataType, Pa
495512

496513
private static Page page(BlockFactory blockFactory, String grouping, String blockType) {
497514
Block dataBlock = dataBlock(blockFactory, blockType);
498-
if (grouping.equals("none")) {
515+
if (grouping.equals(NONE)) {
499516
return new Page(dataBlock);
500517
}
501518
List<Block> blocks = groupingBlocks(grouping, blockType);
@@ -564,7 +581,7 @@ private static Block groupingBlock(String grouping, String blockType) {
564581
default -> throw new UnsupportedOperationException("bad grouping [" + grouping + "]");
565582
};
566583
return switch (grouping) {
567-
case LONGS -> {
584+
case TOP_N_LONGS, LONGS -> {
568585
var builder = blockFactory.newLongBlockBuilder(BLOCK_LENGTH);
569586
for (int i = 0; i < BLOCK_LENGTH; i++) {
570587
for (int v = 0; v < valuesPerGroup; v++) {

‎build-tools-internal/gradle/wrapper/gradle-wrapper.properties‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
distributionBase=GRADLE_USER_HOME
22
distributionPath=wrapper/dists
3-
distributionSha256Sum=d7042b3c11565c192041fc8c4703f541b888286404b4f267138c1d094d8ecdca
4-
distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.1-all.zip
3+
distributionSha256Sum=443c9c8ee2ac1ee0e11881a40f2376d79c66386264a44b24a9f8ca67e633375f
4+
distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.2-all.zip
55
networkTimeout=10000
66
validateDistributionUrl=true
77
zipStoreBase=GRADLE_USER_HOME

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,16 @@
99

1010
package org.elasticsearch.gradle.internal
1111

12+
import spock.lang.IgnoreIf
13+
1214
import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
1315
import org.gradle.testkit.runner.TaskOutcome
1416
import org.xmlunit.builder.DiffBuilder
1517
import org.xmlunit.builder.Input
1618

19+
// Ignoring this test on windows due to what appears to be a bug in the gradle testkit runner.
20+
// https://github.com/elastic/elasticsearch/issues/129100
21+
@IgnoreIf({ os.isWindows() })
1722
class PublishPluginFuncTest extends AbstractGradleFuncTest {
1823

1924
def setup() {

‎build-tools-internal/src/main/groovy/elasticsearch.bc-upgrade-test.gradle‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
import org.elasticsearch.gradle.Version
11+
import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
12+
13+
tasks.register("bcUpgradeTest", StandaloneRestIntegTestTask) {
14+
// We use a phony version here as the real version is provided via `tests.bwc.main.version` system property
15+
usesBwcDistribution(Version.fromString("0.0.0"))
16+
systemProperty("tests.old_cluster_version", "0.0.0")
17+
onlyIf("tests.bwc.main.version system property exists") { System.getProperty("tests.bwc.main.version") != null }
18+
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoPlugin.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,13 @@
1515
import org.gradle.api.Project;
1616
import org.gradle.api.artifacts.Configuration;
1717
import org.gradle.api.attributes.Category;
18+
import org.gradle.api.attributes.Usage;
1819
import org.gradle.api.plugins.JavaPlugin;
1920

2021
public class DependenciesInfoPlugin implements Plugin<Project> {
22+
23+
public static String USAGE_ATTRIBUTE = "DependenciesInfo";
24+
2125
@Override
2226
public void apply(final Project project) {
2327
project.getPlugins().apply(CompileOnlyResolvePlugin.class);
@@ -43,6 +47,9 @@ public void apply(final Project project) {
4347
)
4448
);
4549

50+
dependenciesInfoFilesConfiguration.attributes(
51+
attributes -> attributes.attribute(Usage.USAGE_ATTRIBUTE, project.getObjects().named(Usage.class, USAGE_ATTRIBUTE))
52+
);
4653
project.getArtifacts().add("dependenciesInfoFiles", depsInfo);
4754

4855
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/MethodReplacement.java‎ renamed to ‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/MethodReplacement.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
* License v3.0 only", or the "Server Side Public License, v 1".
88
*/
99

10-
package org.elasticsearch.gradle.internal.dependencies.patches.hdfs;
10+
package org.elasticsearch.gradle.internal.dependencies.patches;
1111

1212
import org.objectweb.asm.MethodVisitor;
1313
import org.objectweb.asm.Opcodes;
@@ -16,7 +16,7 @@ public class MethodReplacement extends MethodVisitor {
1616
private final MethodVisitor delegate;
1717
private final Runnable bodyWriter;
1818

19-
MethodReplacement(MethodVisitor delegate, Runnable bodyWriter) {
19+
public MethodReplacement(MethodVisitor delegate, Runnable bodyWriter) {
2020
super(Opcodes.ASM9);
2121
this.delegate = delegate;
2222
this.bodyWriter = bodyWriter;

0 commit comments

Comments
 (0)