Skip to content

Commit 635c650

Browse files
pawankartik-elasticpawankartik-elastic
authored
Merge branch 'main' into pkar/cps-search-force-reconnect
2 parents 3354ff7 + 8682c18 commit 635c650

File tree

558 files changed

+21271
-7910
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

558 files changed

+21271
-7910
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java‎

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,8 @@ public class ValuesSourceReaderBenchmark {
9292
"double",
9393
"keyword",
9494
"stored_keyword",
95-
"3_stored_keywords" };
95+
"3_stored_keywords",
96+
"keyword_mv" };
9697

9798
private static final int BLOCK_LENGTH = 16 * 1024;
9899
private static final int INDEX_SIZE = 10 * BLOCK_LENGTH;
@@ -332,7 +333,7 @@ public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() {
332333
@Param({ "in_order", "shuffled" })
333334
public String layout;
334335

335-
@Param({ "long", "keyword", "stored_keyword" })
336+
@Param({ "long", "keyword", "stored_keyword", "keyword_mv" })
336337
public String name;
337338

338339
private Directory directory;
@@ -398,6 +399,22 @@ public void benchmark() {
398399
}
399400
}
400401
}
402+
case "keyword_mv" -> {
403+
BytesRef scratch = new BytesRef();
404+
BytesRefBlock values = op.getOutput().<BytesRefBlock>getBlock(1);
405+
for (int p = 0; p < values.getPositionCount(); p++) {
406+
int count = values.getValueCount(p);
407+
if (count > 0) {
408+
int first = values.getFirstValueIndex(p);
409+
for (int i = 0; i < count; i++) {
410+
BytesRef r = values.getBytesRef(first + i, scratch);
411+
r.offset++;
412+
r.length--;
413+
sum += Integer.parseInt(r.utf8ToString());
414+
}
415+
}
416+
}
417+
}
401418
}
402419
}
403420
long expected = 0;
@@ -407,6 +424,16 @@ public void benchmark() {
407424
expected += i % 1000;
408425
}
409426
break;
427+
case "keyword_mv":
428+
for (int i = 0; i < INDEX_SIZE; i++) {
429+
int v1 = i % 1000;
430+
expected += v1;
431+
int v2 = i % 500;
432+
if (v1 != v2) {
433+
expected += v2;
434+
}
435+
}
436+
break;
410437
case "3_stored_keywords":
411438
for (int i = 0; i < INDEX_SIZE; i++) {
412439
expected += 3 * (i % 1000);
@@ -461,7 +488,9 @@ private void setupIndex() throws IOException {
461488
new StoredField("double", (double) i),
462489
new KeywordFieldMapper.KeywordField("keyword_1", new BytesRef(c + i % 1000), keywordFieldType),
463490
new KeywordFieldMapper.KeywordField("keyword_2", new BytesRef(c + i % 1000), keywordFieldType),
464-
new KeywordFieldMapper.KeywordField("keyword_3", new BytesRef(c + i % 1000), keywordFieldType)
491+
new KeywordFieldMapper.KeywordField("keyword_3", new BytesRef(c + i % 1000), keywordFieldType),
492+
new KeywordFieldMapper.KeywordField("keyword_mv", new BytesRef(c + i % 1000), keywordFieldType),
493+
new KeywordFieldMapper.KeywordField("keyword_mv", new BytesRef(c + i % 500), keywordFieldType)
465494
)
466495
);
467496
if (i % COMMIT_INTERVAL == 0) {

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ private static Operator operator(DriverContext driverContext, String grouping, S
191191
new BlockHash.GroupSpec(2, ElementType.BYTES_REF)
192192
);
193193
case TOP_N_LONGS -> List.of(
194-
new BlockHash.GroupSpec(0, ElementType.LONG, false, new BlockHash.TopNDef(0, true, true, TOP_N_LIMIT))
194+
new BlockHash.GroupSpec(0, ElementType.LONG, null, new BlockHash.TopNDef(0, true, true, TOP_N_LIMIT))
195195
);
196196
default -> throw new IllegalArgumentException("unsupported grouping [" + grouping + "]");
197197
};

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesAggregatorBenchmark.java‎

Lines changed: 5 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,7 @@ static void selfTest() {
9595
try {
9696
for (String groups : ValuesAggregatorBenchmark.class.getField("groups").getAnnotationsByType(Param.class)[0].value()) {
9797
for (String dataType : ValuesAggregatorBenchmark.class.getField("dataType").getAnnotationsByType(Param.class)[0].value()) {
98-
run(Integer.parseInt(groups), dataType, 10, 0);
99-
run(Integer.parseInt(groups), dataType, 10, 1);
98+
run(Integer.parseInt(groups), dataType, 10);
10099
}
101100
}
102101
} catch (NoSuchFieldException e) {
@@ -114,10 +113,7 @@ static void selfTest() {
114113
@Param({ BYTES_REF, INT, LONG })
115114
public String dataType;
116115

117-
@Param({ "0", "1" })
118-
public int numOrdinalMerges;
119-
120-
private static Operator operator(DriverContext driverContext, int groups, String dataType, int numOrdinalMerges) {
116+
private static Operator operator(DriverContext driverContext, int groups, String dataType) {
121117
if (groups == 1) {
122118
return new AggregationOperator(
123119
List.of(supplier(dataType).aggregatorFactory(AggregatorMode.SINGLE, List.of(0)).apply(driverContext)),
@@ -132,20 +128,8 @@ private static Operator operator(DriverContext driverContext, int groups, String
132128
) {
133129
@Override
134130
public Page getOutput() {
135-
mergeOrdinal();
136131
return super.getOutput();
137132
}
138-
139-
// simulate OrdinalsGroupingOperator
140-
void mergeOrdinal() {
141-
var merged = supplier(dataType).groupingAggregatorFactory(AggregatorMode.SINGLE, List.of(1)).apply(driverContext);
142-
for (int i = 0; i < numOrdinalMerges; i++) {
143-
for (int p = 0; p < groups; p++) {
144-
merged.addIntermediateRow(p, aggregators.getFirst(), p);
145-
}
146-
}
147-
aggregators.set(0, merged);
148-
}
149133
};
150134
}
151135

@@ -352,12 +336,12 @@ private static Block groupingBlock(int groups) {
352336

353337
@Benchmark
354338
public void run() {
355-
run(groups, dataType, OP_COUNT, numOrdinalMerges);
339+
run(groups, dataType, OP_COUNT);
356340
}
357341

358-
private static void run(int groups, String dataType, int opCount, int numOrdinalMerges) {
342+
private static void run(int groups, String dataType, int opCount) {
359343
DriverContext driverContext = driverContext();
360-
try (Operator operator = operator(driverContext, groups, dataType, numOrdinalMerges)) {
344+
try (Operator operator = operator(driverContext, groups, dataType)) {
361345
Page page = page(groups, dataType);
362346
for (int i = 0; i < opCount; i++) {
363347
operator.addInput(page.shallowCopy());

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java‎

Lines changed: 41 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import java.io.File;
3535
import java.util.List;
3636
import java.util.Map;
37+
import java.util.Set;
3738
import java.util.stream.Stream;
3839

3940
import javax.inject.Inject;
@@ -50,6 +51,8 @@ public abstract class ElasticsearchTestBasePlugin implements Plugin<Project> {
5051

5152
public static final String DUMP_OUTPUT_ON_FAILURE_PROP_NAME = "dumpOutputOnFailure";
5253

54+
public static final Set<String> TEST_TASKS_WITH_ENTITLEMENTS = Set.of("test", "internalClusterTest");
55+
5356
@Inject
5457
protected abstract ProviderFactory getProviderFactory();
5558

@@ -174,14 +177,23 @@ public void execute(Task t) {
174177
nonInputProperties.systemProperty("workspace.dir", Util.locateElasticsearchWorkspace(project.getGradle()));
175178
// we use 'temp' relative to CWD since this is per JVM and tests are forbidden from writing to CWD
176179
nonInputProperties.systemProperty("java.io.tmpdir", test.getWorkingDir().toPath().resolve("temp"));
180+
if (test.getName().equals("internalClusterTest")) {
181+
// configure a node home directory independent of the Java temp dir so that entitlements can be properly enforced
182+
nonInputProperties.systemProperty("tempDir", test.getWorkingDir().toPath().resolve("nodesTemp"));
183+
}
177184

178185
SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
179186
SourceSet mainSourceSet = sourceSets.findByName(SourceSet.MAIN_SOURCE_SET_NAME);
180187
SourceSet testSourceSet = sourceSets.findByName(SourceSet.TEST_SOURCE_SET_NAME);
181-
if ("test".equals(test.getName()) && mainSourceSet != null && testSourceSet != null) {
188+
SourceSet internalClusterTestSourceSet = sourceSets.findByName("internalClusterTest");
189+
190+
if (TEST_TASKS_WITH_ENTITLEMENTS.contains(test.getName()) && mainSourceSet != null && testSourceSet != null) {
182191
FileCollection mainRuntime = mainSourceSet.getRuntimeClasspath();
183192
FileCollection testRuntime = testSourceSet.getRuntimeClasspath();
184-
FileCollection testOnlyFiles = testRuntime.minus(mainRuntime);
193+
FileCollection internalClusterTestRuntime = ("internalClusterTest".equals(test.getName())
194+
&& internalClusterTestSourceSet != null) ? internalClusterTestSourceSet.getRuntimeClasspath() : project.files();
195+
FileCollection testOnlyFiles = testRuntime.plus(internalClusterTestRuntime).minus(mainRuntime);
196+
185197
test.doFirst(task -> test.environment("es.entitlement.testOnlyPath", testOnlyFiles.getAsPath()));
186198
}
187199

@@ -241,14 +253,15 @@ public void execute(Task t) {
241253
* Computes and sets the {@code --patch-module=java.base} and {@code --add-opens=java.base} JVM command line options.
242254
*/
243255
private void configureJavaBaseModuleOptions(Project project) {
244-
project.getTasks().withType(Test.class).matching(task -> task.getName().equals("test")).configureEach(test -> {
245-
FileCollection patchedImmutableCollections = patchedImmutableCollections(project);
256+
project.getTasks().withType(Test.class).configureEach(test -> {
257+
// patch immutable collections only for "test" task
258+
FileCollection patchedImmutableCollections = test.getName().equals("test") ? patchedImmutableCollections(project) : null;
246259
if (patchedImmutableCollections != null) {
247260
test.getInputs().files(patchedImmutableCollections);
248261
test.systemProperty("tests.hackImmutableCollections", "true");
249262
}
250263

251-
FileCollection entitlementBridge = entitlementBridge(project);
264+
FileCollection entitlementBridge = TEST_TASKS_WITH_ENTITLEMENTS.contains(test.getName()) ? entitlementBridge(project) : null;
252265
if (entitlementBridge != null) {
253266
test.getInputs().files(entitlementBridge);
254267
}
@@ -312,27 +325,30 @@ private static void configureEntitlements(Project project) {
312325
}
313326
FileCollection bridgeFiles = bridgeConfig;
314327

315-
project.getTasks().withType(Test.class).configureEach(test -> {
316-
// See also SystemJvmOptions.maybeAttachEntitlementAgent.
317-
318-
// Agent
319-
if (agentFiles.isEmpty() == false) {
320-
test.getInputs().files(agentFiles);
321-
test.systemProperty("es.entitlement.agentJar", agentFiles.getAsPath());
322-
test.systemProperty("jdk.attach.allowAttachSelf", true);
323-
}
328+
project.getTasks()
329+
.withType(Test.class)
330+
.matching(test -> TEST_TASKS_WITH_ENTITLEMENTS.contains(test.getName()))
331+
.configureEach(test -> {
332+
// See also SystemJvmOptions.maybeAttachEntitlementAgent.
333+
334+
// Agent
335+
if (agentFiles.isEmpty() == false) {
336+
test.getInputs().files(agentFiles);
337+
test.systemProperty("es.entitlement.agentJar", agentFiles.getAsPath());
338+
test.systemProperty("jdk.attach.allowAttachSelf", true);
339+
}
324340

325-
// Bridge
326-
if (bridgeFiles.isEmpty() == false) {
327-
String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
328-
test.getInputs().files(bridgeFiles);
329-
// Tests may not be modular, but the JDK still is
330-
test.jvmArgs(
331-
"--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
332-
+ modulesContainingEntitlementInstrumentation
333-
);
334-
}
335-
});
341+
// Bridge
342+
if (bridgeFiles.isEmpty() == false) {
343+
String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
344+
test.getInputs().files(bridgeFiles);
345+
// Tests may not be modular, but the JDK still is
346+
test.jvmArgs(
347+
"--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
348+
+ modulesContainingEntitlementInstrumentation
349+
);
350+
}
351+
});
336352
}
337353

338354
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
5656
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:third-party:jira");
5757
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:third-party:pagerduty");
5858
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:third-party:slack");
59-
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:async-search:qa:rest");
60-
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:autoscaling:qa:rest");
6159
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:deprecation:qa:early-deprecation-rest");
6260
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:deprecation:qa:rest");
6361
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:downsample:qa:with-security");

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/Awsv2ClassPatcher.java‎

Lines changed: 0 additions & 61 deletions
This file was deleted.

0 commit comments

Comments
 (0)