Skip to content

Commit d4981a4

Browse files
ywangdywangd
authored
Rename shard heap to estimated heap (#129514)
Relates: ES-11445
1 parent e7409a8 commit d4981a4

File tree

13 files changed

+98
-94
lines changed

13 files changed

+98
-94
lines changed

‎server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@
1919
import org.elasticsearch.cluster.ClusterInfoService;
2020
import org.elasticsearch.cluster.ClusterInfoServiceUtils;
2121
import org.elasticsearch.cluster.ClusterState;
22+
import org.elasticsearch.cluster.EstimatedHeapUsage;
23+
import org.elasticsearch.cluster.EstimatedHeapUsageCollector;
2224
import org.elasticsearch.cluster.InternalClusterInfoService;
23-
import org.elasticsearch.cluster.ShardHeapUsage;
24-
import org.elasticsearch.cluster.ShardHeapUsageCollector;
2525
import org.elasticsearch.cluster.metadata.IndexMetadata;
2626
import org.elasticsearch.cluster.node.DiscoveryNode;
2727
import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
@@ -123,7 +123,7 @@ public class IndexShardIT extends ESSingleNodeTestCase {
123123

124124
@Override
125125
protected Collection<Class<? extends Plugin>> getPlugins() {
126-
return pluginList(InternalSettingsPlugin.class, BogusShardHeapUsagePlugin.class);
126+
return pluginList(InternalSettingsPlugin.class, BogusEstimatedHeapUsagePlugin.class);
127127
}
128128

129129
public void testLockTryingToDelete() throws Exception {
@@ -264,31 +264,31 @@ public void testExpectedShardSizeIsPresent() throws InterruptedException {
264264
public void testHeapUsageEstimateIsPresent() {
265265
InternalClusterInfoService clusterInfoService = (InternalClusterInfoService) getInstanceFromNode(ClusterInfoService.class);
266266
ClusterInfoServiceUtils.refresh(clusterInfoService);
267-
Map<String, ShardHeapUsage> shardHeapUsages = clusterInfoService.getClusterInfo().getShardHeapUsages();
268-
assertNotNull(shardHeapUsages);
267+
Map<String, EstimatedHeapUsage> estimatedHeapUsages = clusterInfoService.getClusterInfo().getEstimatedHeapUsages();
268+
assertNotNull(estimatedHeapUsages);
269269
// Not collecting yet because it is disabled
270-
assertTrue(shardHeapUsages.isEmpty());
270+
assertTrue(estimatedHeapUsages.isEmpty());
271271

272-
// Enable collection for shard heap usages
272+
// Enable collection for estimated heap usages
273273
updateClusterSettings(
274274
Settings.builder()
275-
.put(InternalClusterInfoService.CLUSTER_ROUTING_ALLOCATION_SHARD_HEAP_THRESHOLD_DECIDER_ENABLED.getKey(), true)
275+
.put(InternalClusterInfoService.CLUSTER_ROUTING_ALLOCATION_ESTIMATED_HEAP_THRESHOLD_DECIDER_ENABLED.getKey(), true)
276276
.build()
277277
);
278278
try {
279279
ClusterInfoServiceUtils.refresh(clusterInfoService);
280280
ClusterState state = getInstanceFromNode(ClusterService.class).state();
281-
shardHeapUsages = clusterInfoService.getClusterInfo().getShardHeapUsages();
282-
assertEquals(state.nodes().size(), shardHeapUsages.size());
281+
estimatedHeapUsages = clusterInfoService.getClusterInfo().getEstimatedHeapUsages();
282+
assertEquals(state.nodes().size(), estimatedHeapUsages.size());
283283
for (DiscoveryNode node : state.nodes()) {
284-
assertTrue(shardHeapUsages.containsKey(node.getId()));
285-
ShardHeapUsage shardHeapUsage = shardHeapUsages.get(node.getId());
286-
assertThat(shardHeapUsage.estimatedFreeBytes(), lessThanOrEqualTo(shardHeapUsage.totalBytes()));
284+
assertTrue(estimatedHeapUsages.containsKey(node.getId()));
285+
EstimatedHeapUsage estimatedHeapUsage = estimatedHeapUsages.get(node.getId());
286+
assertThat(estimatedHeapUsage.estimatedFreeBytes(), lessThanOrEqualTo(estimatedHeapUsage.totalBytes()));
287287
}
288288
} finally {
289289
updateClusterSettings(
290290
Settings.builder()
291-
.putNull(InternalClusterInfoService.CLUSTER_ROUTING_ALLOCATION_SHARD_HEAP_THRESHOLD_DECIDER_ENABLED.getKey())
291+
.putNull(InternalClusterInfoService.CLUSTER_ROUTING_ALLOCATION_ESTIMATED_HEAP_THRESHOLD_DECIDER_ENABLED.getKey())
292292
.build()
293293
);
294294
}
@@ -838,11 +838,11 @@ private static void assertAllIndicesRemovedAndDeletionCompleted(Iterable<Indices
838838
}
839839
}
840840

841-
public static class BogusShardShardHeapUsageCollector implements ShardHeapUsageCollector {
841+
public static class BogusEstimatedEstimatedHeapUsageCollector implements EstimatedHeapUsageCollector {
842842

843-
private final BogusShardHeapUsagePlugin plugin;
843+
private final BogusEstimatedHeapUsagePlugin plugin;
844844

845-
public BogusShardShardHeapUsageCollector(BogusShardHeapUsagePlugin plugin) {
845+
public BogusEstimatedEstimatedHeapUsageCollector(BogusEstimatedHeapUsagePlugin plugin) {
846846
this.plugin = plugin;
847847
}
848848

@@ -859,7 +859,7 @@ public void collectClusterHeapUsage(ActionListener<Map<String, Long>> listener)
859859
}
860860
}
861861

862-
public static class BogusShardHeapUsagePlugin extends Plugin implements ClusterPlugin {
862+
public static class BogusEstimatedHeapUsagePlugin extends Plugin implements ClusterPlugin {
863863

864864
private final SetOnce<ClusterService> clusterService = new SetOnce<>();
865865

‎server/src/internalClusterTest/resources/META-INF/services/org.elasticsearch.cluster.ShardHeapUsageCollector‎ renamed to ‎server/src/internalClusterTest/resources/META-INF/services/org.elasticsearch.cluster.EstimatedHeapUsageCollector‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,4 @@
77
# License v3.0 only", or the "Server Side Public License, v 1".
88
#
99

10-
org.elasticsearch.index.shard.IndexShardIT$BogusShardShardHeapUsageCollector
10+
org.elasticsearch.index.shard.IndexShardIT$BogusEstimatedEstimatedHeapUsageCollector

‎server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ public class ClusterInfo implements ChunkedToXContent, Writeable {
5757
final Map<ShardId, Long> shardDataSetSizes;
5858
final Map<NodeAndShard, String> dataPath;
5959
final Map<NodeAndPath, ReservedSpace> reservedSpace;
60-
final Map<String, ShardHeapUsage> shardHeapUsages;
60+
final Map<String, EstimatedHeapUsage> estimatedHeapUsages;
6161

6262
protected ClusterInfo() {
6363
this(Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), Map.of());
@@ -72,7 +72,7 @@ protected ClusterInfo() {
7272
* @param shardDataSetSizes a shard id to data set size in bytes mapping per shard
7373
* @param dataPath the shard routing to datapath mapping
7474
* @param reservedSpace reserved space per shard broken down by node and data path
75-
* @param shardHeapUsages shard heap usage broken down by node
75+
* @param estimatedHeapUsages estimated heap usage broken down by node
7676
* @see #shardIdentifierFromRouting
7777
*/
7878
public ClusterInfo(
@@ -82,15 +82,15 @@ public ClusterInfo(
8282
Map<ShardId, Long> shardDataSetSizes,
8383
Map<NodeAndShard, String> dataPath,
8484
Map<NodeAndPath, ReservedSpace> reservedSpace,
85-
Map<String, ShardHeapUsage> shardHeapUsages
85+
Map<String, EstimatedHeapUsage> estimatedHeapUsages
8686
) {
8787
this.leastAvailableSpaceUsage = Map.copyOf(leastAvailableSpaceUsage);
8888
this.mostAvailableSpaceUsage = Map.copyOf(mostAvailableSpaceUsage);
8989
this.shardSizes = Map.copyOf(shardSizes);
9090
this.shardDataSetSizes = Map.copyOf(shardDataSetSizes);
9191
this.dataPath = Map.copyOf(dataPath);
9292
this.reservedSpace = Map.copyOf(reservedSpace);
93-
this.shardHeapUsages = Map.copyOf(shardHeapUsages);
93+
this.estimatedHeapUsages = Map.copyOf(estimatedHeapUsages);
9494
}
9595

9696
public ClusterInfo(StreamInput in) throws IOException {
@@ -103,9 +103,9 @@ public ClusterInfo(StreamInput in) throws IOException {
103103
: in.readImmutableMap(nested -> NodeAndShard.from(new ShardRouting(nested)), StreamInput::readString);
104104
this.reservedSpace = in.readImmutableMap(NodeAndPath::new, ReservedSpace::new);
105105
if (in.getTransportVersion().onOrAfter(TransportVersions.HEAP_USAGE_IN_CLUSTER_INFO)) {
106-
this.shardHeapUsages = in.readImmutableMap(ShardHeapUsage::new);
106+
this.estimatedHeapUsages = in.readImmutableMap(EstimatedHeapUsage::new);
107107
} else {
108-
this.shardHeapUsages = Map.of();
108+
this.estimatedHeapUsages = Map.of();
109109
}
110110
}
111111

@@ -122,7 +122,7 @@ public void writeTo(StreamOutput out) throws IOException {
122122
}
123123
out.writeMap(this.reservedSpace);
124124
if (out.getTransportVersion().onOrAfter(TransportVersions.HEAP_USAGE_IN_CLUSTER_INFO)) {
125-
out.writeMap(this.shardHeapUsages, StreamOutput::writeWriteable);
125+
out.writeMap(this.estimatedHeapUsages, StreamOutput::writeWriteable);
126126
}
127127
}
128128

@@ -204,7 +204,7 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
204204
return builder.endObject(); // NodeAndPath
205205
}),
206206
endArray() // end "reserved_sizes"
207-
// NOTE: We don't serialize shardHeapUsages at this stage, to avoid
207+
// NOTE: We don't serialize estimatedHeapUsages at this stage, to avoid
208208
// committing to API payloads until the feature is settled
209209
);
210210
}
@@ -216,8 +216,8 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
216216
* Also note that the map may not be complete, it may contain none, or a subset of the nodes in
217217
* the cluster at any time. It may also contain entries for nodes that have since left the cluster.
218218
*/
219-
public Map<String, ShardHeapUsage> getShardHeapUsages() {
220-
return shardHeapUsages;
219+
public Map<String, EstimatedHeapUsage> getEstimatedHeapUsages() {
220+
return estimatedHeapUsages;
221221
}
222222

223223
/**

‎server/src/main/java/org/elasticsearch/cluster/ClusterInfoSimulator.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ public class ClusterInfoSimulator {
3333
private final CopyOnFirstWriteMap<String, Long> shardSizes;
3434
private final Map<ShardId, Long> shardDataSetSizes;
3535
private final Map<NodeAndShard, String> dataPath;
36-
private final Map<String, ShardHeapUsage> shardHeapUsages;
36+
private final Map<String, EstimatedHeapUsage> estimatedHeapUsages;
3737

3838
public ClusterInfoSimulator(RoutingAllocation allocation) {
3939
this.allocation = allocation;
@@ -42,7 +42,7 @@ public ClusterInfoSimulator(RoutingAllocation allocation) {
4242
this.shardSizes = new CopyOnFirstWriteMap<>(allocation.clusterInfo().shardSizes);
4343
this.shardDataSetSizes = Map.copyOf(allocation.clusterInfo().shardDataSetSizes);
4444
this.dataPath = Map.copyOf(allocation.clusterInfo().dataPath);
45-
this.shardHeapUsages = allocation.clusterInfo().getShardHeapUsages();
45+
this.estimatedHeapUsages = allocation.clusterInfo().getEstimatedHeapUsages();
4646
}
4747

4848
/**
@@ -156,7 +156,7 @@ public ClusterInfo getClusterInfo() {
156156
shardDataSetSizes,
157157
dataPath,
158158
Map.of(),
159-
shardHeapUsages
159+
estimatedHeapUsages
160160
);
161161
}
162162
}

‎server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java‎ renamed to ‎server/src/main/java/org/elasticsearch/cluster/EstimatedHeapUsage.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,14 @@
1818
/**
1919
* Record representing an estimate of the heap used by allocated shards and ongoing merges on a particular node
2020
*/
21-
public record ShardHeapUsage(String nodeId, long totalBytes, long estimatedUsageBytes) implements Writeable {
21+
public record EstimatedHeapUsage(String nodeId, long totalBytes, long estimatedUsageBytes) implements Writeable {
2222

23-
public ShardHeapUsage {
23+
public EstimatedHeapUsage {
2424
assert totalBytes >= 0;
2525
assert estimatedUsageBytes >= 0;
2626
}
2727

28-
public ShardHeapUsage(StreamInput in) throws IOException {
28+
public EstimatedHeapUsage(StreamInput in) throws IOException {
2929
this(in.readString(), in.readVLong(), in.readVLong());
3030
}
3131

‎server/src/main/java/org/elasticsearch/cluster/ShardHeapUsageCollector.java‎ renamed to ‎server/src/main/java/org/elasticsearch/cluster/EstimatedHeapUsageCollector.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,21 +14,21 @@
1414
import java.util.Map;
1515

1616
/**
17-
* Collect the shard heap usage for each node in the cluster.
17+
* Collect the estimated heap usage for each node in the cluster.
1818
* <p>
1919
* Results are returned as a map of node ID to estimated heap usage in bytes
2020
*
21-
* @see ShardHeapUsage
21+
* @see EstimatedHeapUsage
2222
*/
23-
public interface ShardHeapUsageCollector {
23+
public interface EstimatedHeapUsageCollector {
2424

2525
/**
26-
* This will be used when there is no ShardHeapUsageCollector available
26+
* This will be used when there is no EstimatedHeapUsageCollector available
2727
*/
28-
ShardHeapUsageCollector EMPTY = listener -> listener.onResponse(Map.of());
28+
EstimatedHeapUsageCollector EMPTY = listener -> listener.onResponse(Map.of());
2929

3030
/**
31-
* Collect the shard heap usage for every node in the cluster
31+
* Collect the estimated heap usage for every node in the cluster
3232
*
3333
* @param listener The listener which will receive the results
3434
*/

‎server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java‎

Lines changed: 27 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -83,22 +83,22 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt
8383
Property.NodeScope
8484
);
8585

86-
public static final Setting<Boolean> CLUSTER_ROUTING_ALLOCATION_SHARD_HEAP_THRESHOLD_DECIDER_ENABLED = Setting.boolSetting(
87-
"cluster.routing.allocation.shard_heap.threshold_enabled",
86+
public static final Setting<Boolean> CLUSTER_ROUTING_ALLOCATION_ESTIMATED_HEAP_THRESHOLD_DECIDER_ENABLED = Setting.boolSetting(
87+
"cluster.routing.allocation.estimated_heap.threshold_enabled",
8888
false,
8989
Property.Dynamic,
9090
Property.NodeScope
9191
);
9292

9393
private volatile boolean diskThresholdEnabled;
94-
private volatile boolean shardHeapThresholdEnabled;
94+
private volatile boolean estimatedHeapThresholdEnabled;
9595
private volatile TimeValue updateFrequency;
9696
private volatile TimeValue fetchTimeout;
9797

9898
private volatile Map<String, DiskUsage> leastAvailableSpaceUsages;
9999
private volatile Map<String, DiskUsage> mostAvailableSpaceUsages;
100100
private volatile Map<String, ByteSizeValue> maxHeapPerNode;
101-
private volatile Map<String, Long> shardHeapUsagePerNode;
101+
private volatile Map<String, Long> estimatedHeapUsagePerNode;
102102
private volatile IndicesStatsSummary indicesStatsSummary;
103103

104104
private final ThreadPool threadPool;
@@ -107,7 +107,7 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt
107107

108108
private final Object mutex = new Object();
109109
private final List<ActionListener<ClusterInfo>> nextRefreshListeners = new ArrayList<>();
110-
private final ShardHeapUsageCollector shardHeapUsageCollector;
110+
private final EstimatedHeapUsageCollector estimatedHeapUsageCollector;
111111

112112
private AsyncRefresh currentRefresh;
113113
private RefreshScheduler refreshScheduler;
@@ -118,16 +118,16 @@ public InternalClusterInfoService(
118118
ClusterService clusterService,
119119
ThreadPool threadPool,
120120
Client client,
121-
ShardHeapUsageCollector shardHeapUsageCollector
121+
EstimatedHeapUsageCollector estimatedHeapUsageCollector
122122
) {
123123
this.leastAvailableSpaceUsages = Map.of();
124124
this.mostAvailableSpaceUsages = Map.of();
125125
this.maxHeapPerNode = Map.of();
126-
this.shardHeapUsagePerNode = Map.of();
126+
this.estimatedHeapUsagePerNode = Map.of();
127127
this.indicesStatsSummary = IndicesStatsSummary.EMPTY;
128128
this.threadPool = threadPool;
129129
this.client = client;
130-
this.shardHeapUsageCollector = shardHeapUsageCollector;
130+
this.estimatedHeapUsageCollector = estimatedHeapUsageCollector;
131131
this.updateFrequency = INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.get(settings);
132132
this.fetchTimeout = INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.get(settings);
133133
this.diskThresholdEnabled = DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.get(settings);
@@ -139,17 +139,17 @@ public InternalClusterInfoService(
139139
this::setDiskThresholdEnabled
140140
);
141141
clusterSettings.initializeAndWatch(
142-
CLUSTER_ROUTING_ALLOCATION_SHARD_HEAP_THRESHOLD_DECIDER_ENABLED,
143-
this::setShardHeapThresholdEnabled
142+
CLUSTER_ROUTING_ALLOCATION_ESTIMATED_HEAP_THRESHOLD_DECIDER_ENABLED,
143+
this::setEstimatedHeapThresholdEnabled
144144
);
145145
}
146146

147147
private void setDiskThresholdEnabled(boolean diskThresholdEnabled) {
148148
this.diskThresholdEnabled = diskThresholdEnabled;
149149
}
150150

151-
private void setShardHeapThresholdEnabled(boolean shardHeapThresholdEnabled) {
152-
this.shardHeapThresholdEnabled = shardHeapThresholdEnabled;
151+
private void setEstimatedHeapThresholdEnabled(boolean estimatedHeapThresholdEnabled) {
152+
this.estimatedHeapThresholdEnabled = estimatedHeapThresholdEnabled;
153153
}
154154

155155
private void setFetchTimeout(TimeValue fetchTimeout) {
@@ -202,8 +202,8 @@ void execute() {
202202

203203
try (var ignoredRefs = fetchRefs) {
204204
maybeFetchIndicesStats(diskThresholdEnabled);
205-
maybeFetchNodeStats(diskThresholdEnabled || shardHeapThresholdEnabled);
206-
maybeFetchNodesHeapUsage(shardHeapThresholdEnabled);
205+
maybeFetchNodeStats(diskThresholdEnabled || estimatedHeapThresholdEnabled);
206+
maybeFetchNodesEstimatedHeapUsage(estimatedHeapThresholdEnabled);
207207
}
208208
}
209209

@@ -231,28 +231,28 @@ private void maybeFetchNodeStats(boolean shouldFetch) {
231231
}
232232
}
233233

234-
private void maybeFetchNodesHeapUsage(boolean shouldFetch) {
234+
private void maybeFetchNodesEstimatedHeapUsage(boolean shouldFetch) {
235235
if (shouldFetch) {
236236
try (var ignored = threadPool.getThreadContext().clearTraceContext()) {
237-
fetchNodesHeapUsage();
237+
fetchNodesEstimatedHeapUsage();
238238
}
239239
} else {
240-
logger.trace("skipping collecting shard heap usage from cluster, notifying listeners with empty shard heap usage");
241-
shardHeapUsagePerNode = Map.of();
240+
logger.trace("skipping collecting estimated heap usage from cluster, notifying listeners with empty estimated heap usage");
241+
estimatedHeapUsagePerNode = Map.of();
242242
}
243243
}
244244

245-
private void fetchNodesHeapUsage() {
246-
shardHeapUsageCollector.collectClusterHeapUsage(ActionListener.releaseAfter(new ActionListener<>() {
245+
private void fetchNodesEstimatedHeapUsage() {
246+
estimatedHeapUsageCollector.collectClusterHeapUsage(ActionListener.releaseAfter(new ActionListener<>() {
247247
@Override
248-
public void onResponse(Map<String, Long> currentShardHeapUsages) {
249-
shardHeapUsagePerNode = currentShardHeapUsages;
248+
public void onResponse(Map<String, Long> currentEstimatedHeapUsages) {
249+
estimatedHeapUsagePerNode = currentEstimatedHeapUsages;
250250
}
251251

252252
@Override
253253
public void onFailure(Exception e) {
254254
logger.warn("failed to fetch heap usage for nodes", e);
255-
shardHeapUsagePerNode = Map.of();
255+
estimatedHeapUsagePerNode = Map.of();
256256
}
257257
}, fetchRefs.acquire()));
258258
}
@@ -479,11 +479,11 @@ private boolean shouldRefresh() {
479479
@Override
480480
public ClusterInfo getClusterInfo() {
481481
final IndicesStatsSummary indicesStatsSummary = this.indicesStatsSummary; // single volatile read
482-
final Map<String, ShardHeapUsage> shardHeapUsages = new HashMap<>();
482+
final Map<String, EstimatedHeapUsage> estimatedHeapUsages = new HashMap<>();
483483
maxHeapPerNode.forEach((nodeId, maxHeapSize) -> {
484-
final Long estimatedHeapUsage = shardHeapUsagePerNode.get(nodeId);
484+
final Long estimatedHeapUsage = estimatedHeapUsagePerNode.get(nodeId);
485485
if (estimatedHeapUsage != null) {
486-
shardHeapUsages.put(nodeId, new ShardHeapUsage(nodeId, maxHeapSize.getBytes(), estimatedHeapUsage));
486+
estimatedHeapUsages.put(nodeId, new EstimatedHeapUsage(nodeId, maxHeapSize.getBytes(), estimatedHeapUsage));
487487
}
488488
});
489489
return new ClusterInfo(
@@ -493,7 +493,7 @@ public ClusterInfo getClusterInfo() {
493493
indicesStatsSummary.shardDataSetSizes,
494494
indicesStatsSummary.dataPath,
495495
indicesStatsSummary.reservedSpace,
496-
shardHeapUsages
496+
estimatedHeapUsages
497497
);
498498
}
499499

0 commit comments

Comments
 (0)