Skip to content

Commit 9b7b0ce

Browse files
martijnvgmartijnvg
committed
Merge remote-tracking branch 'es/main' into TrackingPostingsInMemoryBytesCodec_tweak2
2 parents e8c9974 + 64f8209 commit 9b7b0ce

File tree

26 files changed

+937
-215
lines changed

26 files changed

+937
-215
lines changed

‎.buildkite/pipelines/pull-request/performance-benchmark.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
steps:
2+
- label: ":pipeline: TODO"
3+
command: echo TODO

‎.buildkite/pull-requests.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
},
1919
{
2020
"enabled": true,
21-
"pipeline_slug": "elasticsearch-performance-esbench-pr",
21+
"pipeline_slug": "elasticsearch-pull-request-performance-benchmark",
2222
"allow_org_users": true,
2323
"allowed_repo_permissions": [
2424
"admin",

‎catalog-info.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -301,3 +301,39 @@ spec:
301301
Daily:
302302
branch: main
303303
cronline: "@daily"
304+
---
305+
# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
306+
apiVersion: backstage.io/v1alpha1
307+
kind: Resource
308+
metadata:
309+
name: buildkite-pipeline-elasticsearch-pull-request-performance-benchmark
310+
description: Elasticsearch pull request performance benchmark
311+
links:
312+
- title: Pipeline
313+
url: https://buildkite.com/elastic/elasticsearch-pull-request-performance-benchmark
314+
spec:
315+
type: buildkite-pipeline
316+
system: buildkite
317+
owner: group:elasticsearch-team
318+
implementation:
319+
apiVersion: buildkite.elastic.dev/v1
320+
kind: Pipeline
321+
metadata:
322+
description: ":elasticsearch: Runs performance benchmark in the PR context"
323+
name: elasticsearch / pull-request / performance-benchmark
324+
spec:
325+
repository: elastic/elasticsearch
326+
pipeline_file: .buildkite/pipelines/pull-request/performance-benchmark.yml
327+
env:
328+
ELASTIC_PR_COMMENTS_ENABLED: "true"
329+
ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
330+
SLACK_NOTIFICATIONS_CHANNEL: "#es-perf-build"
331+
SLACK_NOTIFICATIONS_ALL_BRANCHES: "false" # TODO: change to true later
332+
teams:
333+
elasticsearch-team: {}
334+
ml-core: {}
335+
everyone:
336+
access_level: BUILD_AND_READ
337+
provider_settings:
338+
build_pull_requests: true
339+
trigger_mode: none

‎docs/changelog/132950.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 132950
2+
summary: Speed up loading keyword fields with index sorts
3+
area: "ES|QL"
4+
type: enhancement
5+
issues: []

‎muted-tests.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -537,6 +537,12 @@ tests:
537537
- class: org.elasticsearch.cluster.ClusterInfoServiceIT
538538
method: testMaxQueueLatenciesInClusterInfo
539539
issue: https://github.com/elastic/elasticsearch/issues/132957
540+
- class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
541+
method: test {p0=search/400_synthetic_source/_doc_count}
542+
issue: https://github.com/elastic/elasticsearch/issues/132965
543+
- class: org.elasticsearch.index.mapper.LongFieldMapperTests
544+
method: testSyntheticSourceWithTranslogSnapshot
545+
issue: https://github.com/elastic/elasticsearch/issues/132964
540546

541547
# Examples:
542548
#

‎server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -347,7 +347,9 @@ public void testClusterInfoIncludesNodeUsageStatsForThreadPools() {
347347
var settings = Settings.builder()
348348
.put(
349349
WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_ENABLED_SETTING.getKey(),
350-
WriteLoadConstraintSettings.WriteLoadDeciderStatus.ENABLED
350+
randomBoolean()
351+
? WriteLoadConstraintSettings.WriteLoadDeciderStatus.ENABLED
352+
: WriteLoadConstraintSettings.WriteLoadDeciderStatus.LOW_THRESHOLD_ONLY
351353
)
352354
// Manually control cluster info refreshes
353355
.put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.getKey(), "60m")

‎server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -319,7 +319,9 @@ public void testNodeWriteLoadsArePresent() {
319319
Settings.builder()
320320
.put(
321321
WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_ENABLED_SETTING.getKey(),
322-
WriteLoadConstraintSettings.WriteLoadDeciderStatus.ENABLED
322+
randomBoolean()
323+
? WriteLoadConstraintSettings.WriteLoadDeciderStatus.ENABLED
324+
: WriteLoadConstraintSettings.WriteLoadDeciderStatus.LOW_THRESHOLD_ONLY
323325
)
324326
.build()
325327
);
@@ -376,7 +378,9 @@ public void testShardWriteLoadsArePresent() {
376378
Settings.builder()
377379
.put(
378380
WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_ENABLED_SETTING.getKey(),
379-
WriteLoadConstraintSettings.WriteLoadDeciderStatus.ENABLED
381+
randomBoolean()
382+
? WriteLoadConstraintSettings.WriteLoadDeciderStatus.ENABLED
383+
: WriteLoadConstraintSettings.WriteLoadDeciderStatus.LOW_THRESHOLD_ONLY
380384
)
381385
.build()
382386
);

‎server/src/main/java/org/elasticsearch/cluster/ClusterModule.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@
6666
import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
6767
import org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider;
6868
import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
69+
import org.elasticsearch.cluster.routing.allocation.decider.WriteLoadConstraintDecider;
6970
import org.elasticsearch.cluster.service.ClusterService;
7071
import org.elasticsearch.common.io.stream.NamedWriteable;
7172
import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
@@ -446,6 +447,7 @@ public static Collection<AllocationDecider> createAllocationDeciders(
446447
addAllocationDecider(deciders, new SnapshotInProgressAllocationDecider());
447448
addAllocationDecider(deciders, new RestoreInProgressAllocationDecider());
448449
addAllocationDecider(deciders, new NodeShutdownAllocationDecider());
450+
addAllocationDecider(deciders, new WriteLoadConstraintDecider(clusterSettings));
449451
addAllocationDecider(deciders, new NodeReplacementAllocationDecider());
450452
addAllocationDecider(deciders, new FilterAllocationDecider(settings, clusterSettings));
451453
addAllocationDecider(deciders, new SameShardAllocationDecider(clusterSettings));

‎server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,7 @@ void execute() {
219219
logger.trace("starting async refresh");
220220

221221
try (var ignoredRefs = fetchRefs) {
222-
maybeFetchIndicesStats(diskThresholdEnabled || writeLoadConstraintEnabled == WriteLoadDeciderStatus.ENABLED);
222+
maybeFetchIndicesStats(diskThresholdEnabled || writeLoadConstraintEnabled.atLeastLowThresholdEnabled());
223223
maybeFetchNodeStats(diskThresholdEnabled || estimatedHeapThresholdEnabled);
224224
maybeFetchNodesEstimatedHeapUsage(estimatedHeapThresholdEnabled);
225225
maybeFetchNodesUsageStatsForThreadPools(writeLoadConstraintEnabled);
@@ -262,7 +262,7 @@ private void maybeFetchNodesEstimatedHeapUsage(boolean shouldFetch) {
262262
}
263263

264264
private void maybeFetchNodesUsageStatsForThreadPools(WriteLoadDeciderStatus writeLoadConstraintEnabled) {
265-
if (writeLoadConstraintEnabled != WriteLoadDeciderStatus.DISABLED) {
265+
if (writeLoadConstraintEnabled.atLeastLowThresholdEnabled()) {
266266
try (var ignored = threadPool.getThreadContext().clearTraceContext()) {
267267
fetchNodesUsageStatsForThreadPools();
268268
}
@@ -313,7 +313,7 @@ private void fetchIndicesStats() {
313313
// This returns the shard sizes on disk
314314
indicesStatsRequest.store(true);
315315
}
316-
if (writeLoadConstraintEnabled == WriteLoadDeciderStatus.ENABLED) {
316+
if (writeLoadConstraintEnabled.atLeastLowThresholdEnabled()) {
317317
// This returns the shard write-loads
318318
indicesStatsRequest.indexing(true);
319319
}

‎server/src/main/java/org/elasticsearch/cluster/routing/ShardMovementWriteLoadSimulator.java‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -88,11 +88,32 @@ private static NodeUsageStatsForThreadPools.ThreadPoolUsageStats replaceWritePoo
8888
.get(ThreadPool.Names.WRITE);
8989
return new NodeUsageStatsForThreadPools.ThreadPoolUsageStats(
9090
writeThreadPoolStats.totalThreadPoolThreads(),
91-
(float) Math.max(
92-
(writeThreadPoolStats.averageThreadPoolUtilization() + (writeLoadDelta / writeThreadPoolStats.totalThreadPoolThreads())),
93-
0.0
91+
updateNodeUtilizationWithShardMovements(
92+
writeThreadPoolStats.averageThreadPoolUtilization(),
93+
(float) writeLoadDelta,
94+
writeThreadPoolStats.totalThreadPoolThreads()
9495
),
9596
writeThreadPoolStats.maxThreadPoolQueueLatencyMillis()
9697
);
9798
}
99+
100+
/**
101+
* The {@code nodeUtilization} is the average utilization per thread for some duration of time. The {@code shardWriteLoadDelta} is the
102+
* sum of shards' total execution time. Dividing the shards total execution time by the number of threads provides the average
103+
* utilization of each write thread for those shards. The change in shard load can then be added to the node utilization.
104+
*
105+
* @param nodeUtilization The current node-level write load percent utilization.
106+
* @param shardWriteLoadDelta The change in shard(s) execution time across all threads. This can be positive or negative depending on
107+
* whether shards were moved onto the node or off of the node.
108+
* @param numberOfWriteThreads The number of threads available in the node's write thread pool.
109+
* @return The new node-level write load percent utilization after adding the shard write load delta.
110+
*/
111+
public static float updateNodeUtilizationWithShardMovements(
112+
float nodeUtilization,
113+
float shardWriteLoadDelta,
114+
int numberOfWriteThreads
115+
) {
116+
float newNodeUtilization = nodeUtilization + (shardWriteLoadDelta / numberOfWriteThreads);
117+
return (float) Math.max(newNodeUtilization, 0.0);
118+
}
98119
}

0 commit comments

Comments
 (0)