Skip to content

Commit 697985a

Browse files
john-wagsterjohn-wagster
committed
Merge branch 'main' into ivf_hkmeans_struc2
2 parents a85c16f + 393f707 commit 697985a

File tree

181 files changed

+7632
-3084
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

181 files changed

+7632
-3084
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java‎

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,8 @@ public class ValuesSourceReaderBenchmark {
9292
"double",
9393
"keyword",
9494
"stored_keyword",
95-
"3_stored_keywords" };
95+
"3_stored_keywords",
96+
"keyword_mv" };
9697

9798
private static final int BLOCK_LENGTH = 16 * 1024;
9899
private static final int INDEX_SIZE = 10 * BLOCK_LENGTH;
@@ -332,7 +333,7 @@ public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() {
332333
@Param({ "in_order", "shuffled" })
333334
public String layout;
334335

335-
@Param({ "long", "keyword", "stored_keyword" })
336+
@Param({ "long", "keyword", "stored_keyword", "keyword_mv" })
336337
public String name;
337338

338339
private Directory directory;
@@ -398,6 +399,22 @@ public void benchmark() {
398399
}
399400
}
400401
}
402+
case "keyword_mv" -> {
403+
BytesRef scratch = new BytesRef();
404+
BytesRefBlock values = op.getOutput().<BytesRefBlock>getBlock(1);
405+
for (int p = 0; p < values.getPositionCount(); p++) {
406+
int count = values.getValueCount(p);
407+
if (count > 0) {
408+
int first = values.getFirstValueIndex(p);
409+
for (int i = 0; i < count; i++) {
410+
BytesRef r = values.getBytesRef(first + i, scratch);
411+
r.offset++;
412+
r.length--;
413+
sum += Integer.parseInt(r.utf8ToString());
414+
}
415+
}
416+
}
417+
}
401418
}
402419
}
403420
long expected = 0;
@@ -407,6 +424,16 @@ public void benchmark() {
407424
expected += i % 1000;
408425
}
409426
break;
427+
case "keyword_mv":
428+
for (int i = 0; i < INDEX_SIZE; i++) {
429+
int v1 = i % 1000;
430+
expected += v1;
431+
int v2 = i % 500;
432+
if (v1 != v2) {
433+
expected += v2;
434+
}
435+
}
436+
break;
410437
case "3_stored_keywords":
411438
for (int i = 0; i < INDEX_SIZE; i++) {
412439
expected += 3 * (i % 1000);
@@ -461,7 +488,9 @@ private void setupIndex() throws IOException {
461488
new StoredField("double", (double) i),
462489
new KeywordFieldMapper.KeywordField("keyword_1", new BytesRef(c + i % 1000), keywordFieldType),
463490
new KeywordFieldMapper.KeywordField("keyword_2", new BytesRef(c + i % 1000), keywordFieldType),
464-
new KeywordFieldMapper.KeywordField("keyword_3", new BytesRef(c + i % 1000), keywordFieldType)
491+
new KeywordFieldMapper.KeywordField("keyword_3", new BytesRef(c + i % 1000), keywordFieldType),
492+
new KeywordFieldMapper.KeywordField("keyword_mv", new BytesRef(c + i % 1000), keywordFieldType),
493+
new KeywordFieldMapper.KeywordField("keyword_mv", new BytesRef(c + i % 500), keywordFieldType)
465494
)
466495
);
467496
if (i % COMMIT_INTERVAL == 0) {

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesAggregatorBenchmark.java‎

Lines changed: 5 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,7 @@ static void selfTest() {
9595
try {
9696
for (String groups : ValuesAggregatorBenchmark.class.getField("groups").getAnnotationsByType(Param.class)[0].value()) {
9797
for (String dataType : ValuesAggregatorBenchmark.class.getField("dataType").getAnnotationsByType(Param.class)[0].value()) {
98-
run(Integer.parseInt(groups), dataType, 10, 0);
99-
run(Integer.parseInt(groups), dataType, 10, 1);
98+
run(Integer.parseInt(groups), dataType, 10);
10099
}
101100
}
102101
} catch (NoSuchFieldException e) {
@@ -114,10 +113,7 @@ static void selfTest() {
114113
@Param({ BYTES_REF, INT, LONG })
115114
public String dataType;
116115

117-
@Param({ "0", "1" })
118-
public int numOrdinalMerges;
119-
120-
private static Operator operator(DriverContext driverContext, int groups, String dataType, int numOrdinalMerges) {
116+
private static Operator operator(DriverContext driverContext, int groups, String dataType) {
121117
if (groups == 1) {
122118
return new AggregationOperator(
123119
List.of(supplier(dataType).aggregatorFactory(AggregatorMode.SINGLE, List.of(0)).apply(driverContext)),
@@ -132,20 +128,8 @@ private static Operator operator(DriverContext driverContext, int groups, String
132128
) {
133129
@Override
134130
public Page getOutput() {
135-
mergeOrdinal();
136131
return super.getOutput();
137132
}
138-
139-
// simulate OrdinalsGroupingOperator
140-
void mergeOrdinal() {
141-
var merged = supplier(dataType).groupingAggregatorFactory(AggregatorMode.SINGLE, List.of(1)).apply(driverContext);
142-
for (int i = 0; i < numOrdinalMerges; i++) {
143-
for (int p = 0; p < groups; p++) {
144-
merged.addIntermediateRow(p, aggregators.getFirst(), p);
145-
}
146-
}
147-
aggregators.set(0, merged);
148-
}
149133
};
150134
}
151135

@@ -352,12 +336,12 @@ private static Block groupingBlock(int groups) {
352336

353337
@Benchmark
354338
public void run() {
355-
run(groups, dataType, OP_COUNT, numOrdinalMerges);
339+
run(groups, dataType, OP_COUNT);
356340
}
357341

358-
private static void run(int groups, String dataType, int opCount, int numOrdinalMerges) {
342+
private static void run(int groups, String dataType, int opCount) {
359343
DriverContext driverContext = driverContext();
360-
try (Operator operator = operator(driverContext, groups, dataType, numOrdinalMerges)) {
344+
try (Operator operator = operator(driverContext, groups, dataType)) {
361345
Page page = page(groups, dataType);
362346
for (int i = 0; i < opCount; i++) {
363347
operator.addInput(page.shallowCopy());

‎build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,6 @@ else if (node.getSettingKeys().contains("telemetry.metrics.enabled") == false) {
303303
if (cliDebug) {
304304
enableCliDebug();
305305
}
306-
enableEntitlements();
307306
}
308307

309308
@TaskAction

‎build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -88,12 +88,4 @@ default void enableCliDebug() {
8888
}
8989
}
9090
}
91-
92-
default void enableEntitlements() {
93-
for (ElasticsearchCluster cluster : getClusters()) {
94-
for (ElasticsearchNode node : cluster.getNodes()) {
95-
node.cliJvmArgs("-Des.entitlements.enabled=true");
96-
}
97-
}
98-
}
9991
}

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111

1212
import org.elasticsearch.common.settings.Settings;
1313
import org.elasticsearch.common.util.concurrent.EsExecutors;
14-
import org.elasticsearch.jdk.RuntimeVersionFeature;
1514

1615
import java.io.IOException;
1716
import java.nio.file.Files;
@@ -85,7 +84,6 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
8584
maybeSetActiveProcessorCount(nodeSettings),
8685
maybeSetReplayFile(distroType, isHotspot),
8786
maybeWorkaroundG1Bug(),
88-
maybeAllowSecurityManager(useEntitlements),
8987
maybeAttachEntitlementAgent(esHome, useEntitlements)
9088
).flatMap(s -> s).toList();
9189
}
@@ -160,14 +158,6 @@ private static Stream<String> maybeWorkaroundG1Bug() {
160158
return Stream.of();
161159
}
162160

163-
private static Stream<String> maybeAllowSecurityManager(boolean useEntitlements) {
164-
if (RuntimeVersionFeature.isSecurityManagerAvailable()) {
165-
// Will become conditional on useEntitlements once entitlements can run without SM
166-
return Stream.of("-Djava.security.manager=allow");
167-
}
168-
return Stream.of();
169-
}
170-
171161
private static Stream<String> maybeAttachEntitlementAgent(Path esHome, boolean useEntitlements) {
172162
if (useEntitlements == false) {
173163
return Stream.empty();
@@ -191,7 +181,6 @@ private static Stream<String> maybeAttachEntitlementAgent(Path esHome, boolean u
191181
// into java.base, we must export the bridge from java.base to these modules, as a comma-separated list
192182
String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
193183
return Stream.of(
194-
"-Des.entitlements.enabled=true",
195184
"-XX:+EnableDynamicAgentLoading",
196185
"-Djdk.attach.allowAttachSelf=true",
197186
"--patch-module=java.base=" + bridgeJar,

‎docs/changelog/128639.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 128639
2+
summary: Substitue `date_trunc` with `round_to` when the pre-calculated rounding points
3+
are available
4+
area: ES|QL
5+
type: enhancement
6+
issues: []

‎docs/changelog/129929.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 129929
2+
summary: Add support for RLIKE (LIST) with pushdown
3+
area: ES|QL
4+
type: enhancement
5+
issues: []

‎docs/changelog/131061.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 131061
2+
summary: Speed up reading multivalued keywords
3+
area: ES|QL
4+
type: enhancement
5+
issues: []

‎docs/changelog/131200.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 131200
2+
summary: Improve lost-increment message in repo analysis
3+
area: Snapshot/Restore
4+
type: enhancement
5+
issues: []

‎docs/reference/query-languages/esql/_snippets/functions/description/score.md‎

Lines changed: 6 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)