Skip to content

Commit 30e5861

Browse files
alexey-ivanov-esalexey-ivanov-es
authored
[9.1] GET /_migration/deprecations doesn't check disk watermarks against correct settings values (elastic#138115) (elastic#138837)
1 parent c3981de commit 30e5861

File tree

8 files changed

+286
-134
lines changed

8 files changed

+286
-134
lines changed

‎docs/changelog/138115.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
pr: 138115
2+
summary: GET /_migration/deprecations doesn't check disk watermarks against correct
3+
settings values
4+
area: Infra/Core
5+
type: bug
6+
issues:
7+
- 137005

‎server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -487,6 +487,12 @@ public ByteSizeValue getFreeBytesThresholdLowStage(ByteSizeValue total) {
487487
return getFreeBytesThreshold(total, lowStageWatermark, lowStageMaxHeadroom);
488488
}
489489

490+
public static ByteSizeValue getFreeBytesThresholdLowStage(ByteSizeValue total, ClusterSettings clusterSettings) {
491+
var lowStageWatermark = clusterSettings.get(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING);
492+
var lowStageMaxHeadroom = clusterSettings.get(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_MAX_HEADROOM_SETTING);
493+
return getFreeBytesThreshold(total, lowStageWatermark, lowStageMaxHeadroom);
494+
}
495+
490496
public ByteSizeValue getFreeBytesThresholdHighStage(ByteSizeValue total) {
491497
return getFreeBytesThreshold(total, highStageWatermark, highStageMaxHeadroom);
492498
}

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecker.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ public void check(Client client, ActionListener<List<DeprecationIssue>> listener
5252
.collect(Collectors.toList());
5353
logger.warn("nodes failed to run deprecation checks: {}", failedNodeIds);
5454
for (FailedNodeException failure : response.failures()) {
55-
logger.debug("node {} failed to run deprecation checks: {}", failure.nodeId(), failure);
55+
logger.atWarn().withThrowable(failure).log("node {} failed to run deprecation checks", failure.nodeId());
5656
}
5757
}
5858
l.onResponse(reduceToDeprecationIssues(response));

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java‎

Lines changed: 76 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,10 @@
1515
import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
1616
import org.elasticsearch.client.internal.OriginSettingClient;
1717
import org.elasticsearch.client.internal.node.NodeClient;
18+
import org.elasticsearch.cluster.ClusterInfo;
19+
import org.elasticsearch.cluster.ClusterInfoService;
1820
import org.elasticsearch.cluster.ClusterState;
21+
import org.elasticsearch.cluster.DiskUsage;
1922
import org.elasticsearch.cluster.block.ClusterBlockException;
2023
import org.elasticsearch.cluster.block.ClusterBlockLevel;
2124
import org.elasticsearch.cluster.metadata.ComponentTemplate;
@@ -24,10 +27,16 @@
2427
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
2528
import org.elasticsearch.cluster.metadata.Metadata;
2629
import org.elasticsearch.cluster.metadata.Template;
30+
import org.elasticsearch.cluster.node.DiscoveryNode;
31+
import org.elasticsearch.cluster.node.DiscoveryNodes;
32+
import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
2733
import org.elasticsearch.cluster.service.ClusterService;
2834
import org.elasticsearch.common.regex.Regex;
35+
import org.elasticsearch.common.settings.ClusterSettings;
2936
import org.elasticsearch.common.settings.Setting;
3037
import org.elasticsearch.common.settings.Settings;
38+
import org.elasticsearch.common.unit.ByteSizeValue;
39+
import org.elasticsearch.common.util.CollectionUtils;
3140
import org.elasticsearch.core.Tuple;
3241
import org.elasticsearch.injection.guice.Inject;
3342
import org.elasticsearch.tasks.Task;
@@ -44,10 +53,14 @@
4453
import java.util.Collections;
4554
import java.util.HashMap;
4655
import java.util.List;
56+
import java.util.Locale;
4757
import java.util.Map;
58+
import java.util.Objects;
4859
import java.util.stream.Collectors;
4960
import java.util.stream.Stream;
5061

62+
import static org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING;
63+
5164
public class TransportDeprecationInfoAction extends TransportMasterNodeReadAction<
5265
DeprecationInfoAction.Request,
5366
DeprecationInfoAction.Response> {
@@ -62,6 +75,7 @@ public class TransportDeprecationInfoAction extends TransportMasterNodeReadActio
6275
private final IndexNameExpressionResolver indexNameExpressionResolver;
6376
private final Settings settings;
6477
private final NamedXContentRegistry xContentRegistry;
78+
private final ClusterInfoService clusterInfoService;
6579
private volatile List<String> skipTheseDeprecations;
6680
private final NodeDeprecationChecker nodeDeprecationChecker;
6781
private final ClusterDeprecationChecker clusterDeprecationChecker;
@@ -76,7 +90,8 @@ public TransportDeprecationInfoAction(
7690
ActionFilters actionFilters,
7791
IndexNameExpressionResolver indexNameExpressionResolver,
7892
NodeClient client,
79-
NamedXContentRegistry xContentRegistry
93+
NamedXContentRegistry xContentRegistry,
94+
ClusterInfoService clusterInfoService
8095
) {
8196
super(
8297
DeprecationInfoAction.NAME,
@@ -92,6 +107,7 @@ public TransportDeprecationInfoAction(
92107
this.indexNameExpressionResolver = indexNameExpressionResolver;
93108
this.settings = settings;
94109
this.xContentRegistry = xContentRegistry;
110+
this.clusterInfoService = clusterInfoService;
95111
skipTheseDeprecations = SKIP_DEPRECATIONS_SETTING.get(settings);
96112
nodeDeprecationChecker = new NodeDeprecationChecker(threadPool);
97113
clusterDeprecationChecker = new ClusterDeprecationChecker(xContentRegistry);
@@ -122,7 +138,13 @@ protected final void masterOperation(
122138
ClusterState state,
123139
final ActionListener<DeprecationInfoAction.Response> listener
124140
) {
125-
PrecomputedData precomputedData = new PrecomputedData();
141+
DeprecationIssue lowWatermarkIssue = checkDiskLowWatermark(
142+
clusterService.getClusterSettings(),
143+
clusterInfoService.getClusterInfo(),
144+
state.nodes()
145+
);
146+
PrecomputedData precomputedData = new PrecomputedData(lowWatermarkIssue);
147+
126148
try (var refs = new RefCountingListener(checkAndCreateResponse(state, request, precomputedData, listener))) {
127149
nodeDeprecationChecker.check(client, refs.acquire(precomputedData::setOnceNodeSettingsIssues));
128150
transformConfigs(refs.acquire(precomputedData::setOnceTransformConfigs));
@@ -149,7 +171,7 @@ protected final void masterOperation(
149171
* @return The listener that should be executed after all the remote requests have completed and the {@link PrecomputedData}
150172
* is initialised.
151173
*/
152-
public ActionListener<Void> checkAndCreateResponse(
174+
private ActionListener<Void> checkAndCreateResponse(
153175
ClusterState state,
154176
DeprecationInfoAction.Request request,
155177
PrecomputedData precomputedData,
@@ -238,6 +260,11 @@ public static class PrecomputedData {
238260
private final SetOnce<List<DeprecationIssue>> nodeSettingsIssues = new SetOnce<>();
239261
private final SetOnce<Map<String, List<DeprecationIssue>>> pluginIssues = new SetOnce<>();
240262
private final SetOnce<List<TransformConfig>> transformConfigs = new SetOnce<>();
263+
private final DeprecationIssue diskWatermarkIssue;
264+
265+
public PrecomputedData(DeprecationIssue diskWatermarkIssue) {
266+
this.diskWatermarkIssue = diskWatermarkIssue;
267+
}
241268

242269
public void setOnceNodeSettingsIssues(List<DeprecationIssue> nodeSettingsIssues) {
243270
this.nodeSettingsIssues.set(nodeSettingsIssues);
@@ -252,7 +279,12 @@ public void setOnceTransformConfigs(List<TransformConfig> transformConfigs) {
252279
}
253280

254281
public List<DeprecationIssue> nodeSettingsIssues() {
255-
return nodeSettingsIssues.get();
282+
List<DeprecationIssue> deprecationIssues = nodeSettingsIssues.get();
283+
assert deprecationIssues != null : "nodeSettingsIssues must be set before calling this method";
284+
if (diskWatermarkIssue == null) {
285+
return deprecationIssues;
286+
}
287+
return CollectionUtils.appendToCopy(deprecationIssues, diskWatermarkIssue);
256288
}
257289

258290
public Map<String, List<DeprecationIssue>> pluginIssues() {
@@ -395,4 +427,44 @@ private void transformConfigs(PageParams currentPage, ActionListener<Stream<Tran
395427
private <T> ActionListener<T> executeInGenericThreadpool(ActionListener<T> listener) {
396428
return new ThreadedActionListener<>(threadPool.generic(), listener);
397429
}
430+
431+
static DeprecationIssue checkDiskLowWatermark(ClusterSettings clusterSettings, ClusterInfo clusterInfo, DiscoveryNodes discoveryNodes) {
432+
Map<String, DiskUsage> nodeMostAvailableDiskUsages = clusterInfo.getNodeMostAvailableDiskUsages();
433+
434+
List<String> impactedNodeNames = nodeMostAvailableDiskUsages.entrySet()
435+
.stream()
436+
.filter(e -> exceedsLowWatermark(clusterSettings, e.getValue()))
437+
.map(Map.Entry::getKey)
438+
.map(discoveryNodes::get)
439+
.filter(Objects::nonNull)
440+
.map(DiscoveryNode::getName)
441+
.sorted()
442+
.toList();
443+
444+
if (impactedNodeNames.isEmpty()) {
445+
return null;
446+
}
447+
448+
return new DeprecationIssue(
449+
DeprecationIssue.Level.CRITICAL,
450+
"Disk usage exceeds low watermark",
451+
"https://ela.st/es-deprecation-7-disk-watermark-exceeded",
452+
String.format(
453+
Locale.ROOT,
454+
"Disk usage exceeds low watermark, which will prevent reindexing indices during upgrade. Get disk usage on "
455+
+ "all nodes below the value specified in %s (nodes impacted: %s)",
456+
CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(),
457+
impactedNodeNames
458+
),
459+
false,
460+
null
461+
);
462+
}
463+
464+
private static boolean exceedsLowWatermark(ClusterSettings clusterSettings, DiskUsage usage) {
465+
long freeBytes = usage.freeBytes();
466+
long totalBytes = usage.totalBytes();
467+
return freeBytes < DiskThresholdSettings.getFreeBytesThresholdLowStage(ByteSizeValue.ofBytes(totalBytes), clusterSettings)
468+
.getBytes();
469+
}
398470
}

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java‎

Lines changed: 1 addition & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -11,19 +11,13 @@
1111
import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
1212
import org.elasticsearch.action.support.ActionFilters;
1313
import org.elasticsearch.action.support.nodes.TransportNodesAction;
14-
import org.elasticsearch.cluster.ClusterInfo;
15-
import org.elasticsearch.cluster.ClusterInfoService;
1614
import org.elasticsearch.cluster.ClusterState;
17-
import org.elasticsearch.cluster.DiskUsage;
1815
import org.elasticsearch.cluster.metadata.Metadata;
1916
import org.elasticsearch.cluster.node.DiscoveryNode;
20-
import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
2117
import org.elasticsearch.cluster.service.ClusterService;
2218
import org.elasticsearch.common.io.stream.StreamInput;
2319
import org.elasticsearch.common.regex.Regex;
24-
import org.elasticsearch.common.settings.ClusterSettings;
2520
import org.elasticsearch.common.settings.Settings;
26-
import org.elasticsearch.common.unit.ByteSizeValue;
2721
import org.elasticsearch.injection.guice.Inject;
2822
import org.elasticsearch.license.XPackLicenseState;
2923
import org.elasticsearch.plugins.PluginsService;
@@ -35,11 +29,8 @@
3529
import java.io.IOException;
3630
import java.util.Collections;
3731
import java.util.List;
38-
import java.util.Locale;
3932
import java.util.Objects;
4033

41-
import static org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING;
42-
4334
public class TransportNodeDeprecationCheckAction extends TransportNodesAction<
4435
NodesDeprecationCheckRequest,
4536
NodesDeprecationCheckResponse,
@@ -50,7 +41,6 @@ public class TransportNodeDeprecationCheckAction extends TransportNodesAction<
5041
private final Settings settings;
5142
private final XPackLicenseState licenseState;
5243
private final PluginsService pluginsService;
53-
private final ClusterInfoService clusterInfoService;
5444
private volatile List<String> skipTheseDeprecations;
5545

5646
@Inject
@@ -61,8 +51,7 @@ public TransportNodeDeprecationCheckAction(
6151
ClusterService clusterService,
6252
TransportService transportService,
6353
PluginsService pluginsService,
64-
ActionFilters actionFilters,
65-
ClusterInfoService clusterInfoService
54+
ActionFilters actionFilters
6655
) {
6756
super(
6857
NodesDeprecationCheckAction.NAME,
@@ -75,7 +64,6 @@ public TransportNodeDeprecationCheckAction(
7564
this.settings = settings;
7665
this.pluginsService = pluginsService;
7766
this.licenseState = licenseState;
78-
this.clusterInfoService = clusterInfoService;
7967
skipTheseDeprecations = TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING.get(settings);
8068
// Safe to register this here because it happens synchronously before the cluster service is started:
8169
clusterService.getClusterSettings()
@@ -135,55 +123,6 @@ NodesDeprecationCheckAction.NodeResponse nodeOperation(
135123
.map(c -> c.apply(filteredNodeSettings, pluginsService.info(), filteredClusterState, licenseState))
136124
.filter(Objects::nonNull)
137125
.toList();
138-
DeprecationIssue watermarkIssue = checkDiskLowWatermark(
139-
filteredNodeSettings,
140-
filteredClusterState.metadata().settings(),
141-
clusterInfoService.getClusterInfo(),
142-
clusterService.getClusterSettings(),
143-
transportService.getLocalNode().getId()
144-
);
145-
if (watermarkIssue != null) {
146-
issues.add(watermarkIssue);
147-
}
148126
return new NodesDeprecationCheckAction.NodeResponse(transportService.getLocalNode(), issues);
149127
}
150-
151-
static DeprecationIssue checkDiskLowWatermark(
152-
Settings nodeSettings,
153-
Settings dynamicSettings,
154-
ClusterInfo clusterInfo,
155-
ClusterSettings clusterSettings,
156-
String nodeId
157-
) {
158-
DiskUsage usage = clusterInfo.getNodeMostAvailableDiskUsages().get(nodeId);
159-
if (usage != null) {
160-
long freeBytes = usage.freeBytes();
161-
long totalBytes = usage.totalBytes();
162-
if (exceedsLowWatermark(nodeSettings, clusterSettings, freeBytes, totalBytes)
163-
|| exceedsLowWatermark(dynamicSettings, clusterSettings, freeBytes, totalBytes)) {
164-
return new DeprecationIssue(
165-
DeprecationIssue.Level.CRITICAL,
166-
"Disk usage exceeds low watermark",
167-
"https://ela.st/es-deprecation-7-disk-watermark-exceeded",
168-
String.format(
169-
Locale.ROOT,
170-
"Disk usage exceeds low watermark, which will prevent reindexing indices during upgrade. Get disk usage on "
171-
+ "all nodes below the value specified in %s",
172-
CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey()
173-
),
174-
false,
175-
null
176-
);
177-
}
178-
}
179-
return null;
180-
}
181-
182-
private static boolean exceedsLowWatermark(Settings settingsToCheck, ClusterSettings clusterSettings, long freeBytes, long totalBytes) {
183-
DiskThresholdSettings diskThresholdSettings = new DiskThresholdSettings(settingsToCheck, clusterSettings);
184-
if (freeBytes < diskThresholdSettings.getFreeBytesThresholdLowStage(ByteSizeValue.ofBytes(totalBytes)).getBytes()) {
185-
return true;
186-
}
187-
return false;
188-
}
189128
}

‎x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,9 @@ public class IndexDeprecationCheckerTests extends ESTestCase {
5151
private static final IndexVersion OLD_VERSION = IndexVersion.fromId(7170099);
5252
private final IndexNameExpressionResolver indexNameExpressionResolver = TestIndexNameExpressionResolver.newInstance();
5353
private final IndexDeprecationChecker checker = new IndexDeprecationChecker(indexNameExpressionResolver);
54-
private final TransportDeprecationInfoAction.PrecomputedData emptyPrecomputedData =
55-
new TransportDeprecationInfoAction.PrecomputedData();
54+
private final TransportDeprecationInfoAction.PrecomputedData emptyPrecomputedData = new TransportDeprecationInfoAction.PrecomputedData(
55+
null
56+
);
5657
private final IndexMetadata.State indexMetdataState;
5758

5859
public IndexDeprecationCheckerTests(@Name("indexMetadataState") IndexMetadata.State indexMetdataState) {
@@ -793,7 +794,7 @@ private TransportDeprecationInfoAction.PrecomputedData createContextWithTransfor
793794
);
794795
}
795796
}
796-
TransportDeprecationInfoAction.PrecomputedData precomputedData = new TransportDeprecationInfoAction.PrecomputedData();
797+
TransportDeprecationInfoAction.PrecomputedData precomputedData = new TransportDeprecationInfoAction.PrecomputedData(null);
797798
precomputedData.setOnceTransformConfigs(transforms);
798799
return precomputedData;
799800
}

0 commit comments

Comments
 (0)