Skip to content

Commit 24f2770

Browse files
benchaplinbenchaplin
committed
Merge branch 'main' into skip_search_shards_with_index_block
2 parents 51d5196 + 929f65b commit 24f2770

File tree

602 files changed

+21286
-8107
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

602 files changed

+21286
-8107
lines changed

‎benchmarks/README.md‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -152,11 +152,10 @@ exit
152152
Grab the async profiler from https://github.com/jvm-profiling-tools/async-profiler
153153
and run `prof async` like so:
154154
```
155-
gradlew -p benchmarks/ run --args 'LongKeyedBucketOrdsBenchmark.multiBucket -prof "async:libPath=/home/nik9000/Downloads/async-profiler-3.0-29ee888-linux-x64/lib/libasyncProfiler.so;dir=/tmp/prof;output=flamegraph"'
155+
gradlew -p benchmarks/ run --args 'LongKeyedBucketOrdsBenchmark.multiBucket -prof "async:libPath=/home/nik9000/Downloads/async-profiler-4.0-linux-x64/lib/libasyncProfiler.so;dir=/tmp/prof;output=flamegraph"'
156156
```
157157

158-
Note: As of January 2025 the latest release of async profiler doesn't work
159-
with our JDK but the nightly is fine.
158+
Note: As of July 2025 the 4.0 release of the async profiler works well.
160159

161160
If you are on Mac, this'll warn you that you downloaded the shared library from
162161
the internet. You'll need to go to settings and allow it to run.

‎benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java‎

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,10 @@
2424
import org.apache.lucene.util.BytesRef;
2525
import org.apache.lucene.util.NumericUtils;
2626
import org.elasticsearch.common.breaker.NoopCircuitBreaker;
27+
import org.elasticsearch.common.logging.LogConfigurator;
2728
import org.elasticsearch.common.lucene.Lucene;
2829
import org.elasticsearch.common.settings.Settings;
30+
import org.elasticsearch.common.unit.ByteSizeValue;
2931
import org.elasticsearch.common.util.BigArrays;
3032
import org.elasticsearch.compute.data.BlockFactory;
3133
import org.elasticsearch.compute.data.BytesRefBlock;
@@ -85,14 +87,19 @@
8587
@State(Scope.Thread)
8688
@Fork(1)
8789
public class ValuesSourceReaderBenchmark {
90+
static {
91+
LogConfigurator.configureESLogging();
92+
}
93+
8894
private static final String[] SUPPORTED_LAYOUTS = new String[] { "in_order", "shuffled", "shuffled_singles" };
8995
private static final String[] SUPPORTED_NAMES = new String[] {
9096
"long",
9197
"int",
9298
"double",
9399
"keyword",
94100
"stored_keyword",
95-
"3_stored_keywords" };
101+
"3_stored_keywords",
102+
"keyword_mv" };
96103

97104
private static final int BLOCK_LENGTH = 16 * 1024;
98105
private static final int INDEX_SIZE = 10 * BLOCK_LENGTH;
@@ -332,7 +339,7 @@ public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() {
332339
@Param({ "in_order", "shuffled" })
333340
public String layout;
334341

335-
@Param({ "long", "keyword", "stored_keyword" })
342+
@Param({ "long", "keyword", "stored_keyword", "keyword_mv" })
336343
public String name;
337344

338345
private Directory directory;
@@ -344,6 +351,7 @@ public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() {
344351
public void benchmark() {
345352
ValuesSourceReaderOperator op = new ValuesSourceReaderOperator(
346353
blockFactory,
354+
ByteSizeValue.ofMb(1).getBytes(),
347355
fields(name),
348356
List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> {
349357
throw new UnsupportedOperationException("can't load _source here");
@@ -398,6 +406,22 @@ public void benchmark() {
398406
}
399407
}
400408
}
409+
case "keyword_mv" -> {
410+
BytesRef scratch = new BytesRef();
411+
BytesRefBlock values = op.getOutput().<BytesRefBlock>getBlock(1);
412+
for (int p = 0; p < values.getPositionCount(); p++) {
413+
int count = values.getValueCount(p);
414+
if (count > 0) {
415+
int first = values.getFirstValueIndex(p);
416+
for (int i = 0; i < count; i++) {
417+
BytesRef r = values.getBytesRef(first + i, scratch);
418+
r.offset++;
419+
r.length--;
420+
sum += Integer.parseInt(r.utf8ToString());
421+
}
422+
}
423+
}
424+
}
401425
}
402426
}
403427
long expected = 0;
@@ -407,6 +431,16 @@ public void benchmark() {
407431
expected += i % 1000;
408432
}
409433
break;
434+
case "keyword_mv":
435+
for (int i = 0; i < INDEX_SIZE; i++) {
436+
int v1 = i % 1000;
437+
expected += v1;
438+
int v2 = i % 500;
439+
if (v1 != v2) {
440+
expected += v2;
441+
}
442+
}
443+
break;
410444
case "3_stored_keywords":
411445
for (int i = 0; i < INDEX_SIZE; i++) {
412446
expected += 3 * (i % 1000);
@@ -461,7 +495,9 @@ private void setupIndex() throws IOException {
461495
new StoredField("double", (double) i),
462496
new KeywordFieldMapper.KeywordField("keyword_1", new BytesRef(c + i % 1000), keywordFieldType),
463497
new KeywordFieldMapper.KeywordField("keyword_2", new BytesRef(c + i % 1000), keywordFieldType),
464-
new KeywordFieldMapper.KeywordField("keyword_3", new BytesRef(c + i % 1000), keywordFieldType)
498+
new KeywordFieldMapper.KeywordField("keyword_3", new BytesRef(c + i % 1000), keywordFieldType),
499+
new KeywordFieldMapper.KeywordField("keyword_mv", new BytesRef(c + i % 1000), keywordFieldType),
500+
new KeywordFieldMapper.KeywordField("keyword_mv", new BytesRef(c + i % 500), keywordFieldType)
465501
)
466502
);
467503
if (i % COMMIT_INTERVAL == 0) {

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ private static Operator operator(DriverContext driverContext, String grouping, S
191191
new BlockHash.GroupSpec(2, ElementType.BYTES_REF)
192192
);
193193
case TOP_N_LONGS -> List.of(
194-
new BlockHash.GroupSpec(0, ElementType.LONG, false, new BlockHash.TopNDef(0, true, true, TOP_N_LIMIT))
194+
new BlockHash.GroupSpec(0, ElementType.LONG, null, new BlockHash.TopNDef(0, true, true, TOP_N_LIMIT))
195195
);
196196
default -> throw new IllegalArgumentException("unsupported grouping [" + grouping + "]");
197197
};

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesAggregatorBenchmark.java‎

Lines changed: 5 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,7 @@ static void selfTest() {
9595
try {
9696
for (String groups : ValuesAggregatorBenchmark.class.getField("groups").getAnnotationsByType(Param.class)[0].value()) {
9797
for (String dataType : ValuesAggregatorBenchmark.class.getField("dataType").getAnnotationsByType(Param.class)[0].value()) {
98-
run(Integer.parseInt(groups), dataType, 10, 0);
99-
run(Integer.parseInt(groups), dataType, 10, 1);
98+
run(Integer.parseInt(groups), dataType, 10);
10099
}
101100
}
102101
} catch (NoSuchFieldException e) {
@@ -114,10 +113,7 @@ static void selfTest() {
114113
@Param({ BYTES_REF, INT, LONG })
115114
public String dataType;
116115

117-
@Param({ "0", "1" })
118-
public int numOrdinalMerges;
119-
120-
private static Operator operator(DriverContext driverContext, int groups, String dataType, int numOrdinalMerges) {
116+
private static Operator operator(DriverContext driverContext, int groups, String dataType) {
121117
if (groups == 1) {
122118
return new AggregationOperator(
123119
List.of(supplier(dataType).aggregatorFactory(AggregatorMode.SINGLE, List.of(0)).apply(driverContext)),
@@ -132,20 +128,8 @@ private static Operator operator(DriverContext driverContext, int groups, String
132128
) {
133129
@Override
134130
public Page getOutput() {
135-
mergeOrdinal();
136131
return super.getOutput();
137132
}
138-
139-
// simulate OrdinalsGroupingOperator
140-
void mergeOrdinal() {
141-
var merged = supplier(dataType).groupingAggregatorFactory(AggregatorMode.SINGLE, List.of(1)).apply(driverContext);
142-
for (int i = 0; i < numOrdinalMerges; i++) {
143-
for (int p = 0; p < groups; p++) {
144-
merged.addIntermediateRow(p, aggregators.getFirst(), p);
145-
}
146-
}
147-
aggregators.set(0, merged);
148-
}
149133
};
150134
}
151135

@@ -352,12 +336,12 @@ private static Block groupingBlock(int groups) {
352336

353337
@Benchmark
354338
public void run() {
355-
run(groups, dataType, OP_COUNT, numOrdinalMerges);
339+
run(groups, dataType, OP_COUNT);
356340
}
357341

358-
private static void run(int groups, String dataType, int opCount, int numOrdinalMerges) {
342+
private static void run(int groups, String dataType, int opCount) {
359343
DriverContext driverContext = driverContext();
360-
try (Operator operator = operator(driverContext, groups, dataType, numOrdinalMerges)) {
344+
try (Operator operator = operator(driverContext, groups, dataType)) {
361345
Page page = page(groups, dataType);
362346
for (int i = 0; i < opCount; i++) {
363347
operator.addInput(page.shallowCopy());

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java‎

Lines changed: 41 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import java.io.File;
3535
import java.util.List;
3636
import java.util.Map;
37+
import java.util.Set;
3738
import java.util.stream.Stream;
3839

3940
import javax.inject.Inject;
@@ -50,6 +51,8 @@ public abstract class ElasticsearchTestBasePlugin implements Plugin<Project> {
5051

5152
public static final String DUMP_OUTPUT_ON_FAILURE_PROP_NAME = "dumpOutputOnFailure";
5253

54+
public static final Set<String> TEST_TASKS_WITH_ENTITLEMENTS = Set.of("test", "internalClusterTest");
55+
5356
@Inject
5457
protected abstract ProviderFactory getProviderFactory();
5558

@@ -174,14 +177,23 @@ public void execute(Task t) {
174177
nonInputProperties.systemProperty("workspace.dir", Util.locateElasticsearchWorkspace(project.getGradle()));
175178
// we use 'temp' relative to CWD since this is per JVM and tests are forbidden from writing to CWD
176179
nonInputProperties.systemProperty("java.io.tmpdir", test.getWorkingDir().toPath().resolve("temp"));
180+
if (test.getName().equals("internalClusterTest")) {
181+
// configure a node home directory independent of the Java temp dir so that entitlements can be properly enforced
182+
nonInputProperties.systemProperty("tempDir", test.getWorkingDir().toPath().resolve("nodesTemp"));
183+
}
177184

178185
SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
179186
SourceSet mainSourceSet = sourceSets.findByName(SourceSet.MAIN_SOURCE_SET_NAME);
180187
SourceSet testSourceSet = sourceSets.findByName(SourceSet.TEST_SOURCE_SET_NAME);
181-
if ("test".equals(test.getName()) && mainSourceSet != null && testSourceSet != null) {
188+
SourceSet internalClusterTestSourceSet = sourceSets.findByName("internalClusterTest");
189+
190+
if (TEST_TASKS_WITH_ENTITLEMENTS.contains(test.getName()) && mainSourceSet != null && testSourceSet != null) {
182191
FileCollection mainRuntime = mainSourceSet.getRuntimeClasspath();
183192
FileCollection testRuntime = testSourceSet.getRuntimeClasspath();
184-
FileCollection testOnlyFiles = testRuntime.minus(mainRuntime);
193+
FileCollection internalClusterTestRuntime = ("internalClusterTest".equals(test.getName())
194+
&& internalClusterTestSourceSet != null) ? internalClusterTestSourceSet.getRuntimeClasspath() : project.files();
195+
FileCollection testOnlyFiles = testRuntime.plus(internalClusterTestRuntime).minus(mainRuntime);
196+
185197
test.doFirst(task -> test.environment("es.entitlement.testOnlyPath", testOnlyFiles.getAsPath()));
186198
}
187199

@@ -241,14 +253,15 @@ public void execute(Task t) {
241253
* Computes and sets the {@code --patch-module=java.base} and {@code --add-opens=java.base} JVM command line options.
242254
*/
243255
private void configureJavaBaseModuleOptions(Project project) {
244-
project.getTasks().withType(Test.class).matching(task -> task.getName().equals("test")).configureEach(test -> {
245-
FileCollection patchedImmutableCollections = patchedImmutableCollections(project);
256+
project.getTasks().withType(Test.class).configureEach(test -> {
257+
// patch immutable collections only for "test" task
258+
FileCollection patchedImmutableCollections = test.getName().equals("test") ? patchedImmutableCollections(project) : null;
246259
if (patchedImmutableCollections != null) {
247260
test.getInputs().files(patchedImmutableCollections);
248261
test.systemProperty("tests.hackImmutableCollections", "true");
249262
}
250263

251-
FileCollection entitlementBridge = entitlementBridge(project);
264+
FileCollection entitlementBridge = TEST_TASKS_WITH_ENTITLEMENTS.contains(test.getName()) ? entitlementBridge(project) : null;
252265
if (entitlementBridge != null) {
253266
test.getInputs().files(entitlementBridge);
254267
}
@@ -312,27 +325,30 @@ private static void configureEntitlements(Project project) {
312325
}
313326
FileCollection bridgeFiles = bridgeConfig;
314327

315-
project.getTasks().withType(Test.class).configureEach(test -> {
316-
// See also SystemJvmOptions.maybeAttachEntitlementAgent.
317-
318-
// Agent
319-
if (agentFiles.isEmpty() == false) {
320-
test.getInputs().files(agentFiles);
321-
test.systemProperty("es.entitlement.agentJar", agentFiles.getAsPath());
322-
test.systemProperty("jdk.attach.allowAttachSelf", true);
323-
}
328+
project.getTasks()
329+
.withType(Test.class)
330+
.matching(test -> TEST_TASKS_WITH_ENTITLEMENTS.contains(test.getName()))
331+
.configureEach(test -> {
332+
// See also SystemJvmOptions.maybeAttachEntitlementAgent.
333+
334+
// Agent
335+
if (agentFiles.isEmpty() == false) {
336+
test.getInputs().files(agentFiles);
337+
test.systemProperty("es.entitlement.agentJar", agentFiles.getAsPath());
338+
test.systemProperty("jdk.attach.allowAttachSelf", true);
339+
}
324340

325-
// Bridge
326-
if (bridgeFiles.isEmpty() == false) {
327-
String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
328-
test.getInputs().files(bridgeFiles);
329-
// Tests may not be modular, but the JDK still is
330-
test.jvmArgs(
331-
"--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
332-
+ modulesContainingEntitlementInstrumentation
333-
);
334-
}
335-
});
341+
// Bridge
342+
if (bridgeFiles.isEmpty() == false) {
343+
String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
344+
test.getInputs().files(bridgeFiles);
345+
// Tests may not be modular, but the JDK still is
346+
test.jvmArgs(
347+
"--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
348+
+ modulesContainingEntitlementInstrumentation
349+
);
350+
}
351+
});
336352
}
337353

338354
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
5656
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:third-party:jira");
5757
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:third-party:pagerduty");
5858
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:third-party:slack");
59-
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:async-search:qa:rest");
60-
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:autoscaling:qa:rest");
6159
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:deprecation:qa:early-deprecation-rest");
6260
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:deprecation:qa:rest");
6361
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:downsample:qa:with-security");

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/Awsv2ClassPatcher.java‎

Lines changed: 0 additions & 61 deletions
This file was deleted.

0 commit comments

Comments
 (0)