From 2678e8f6663a7d1cf1a181968944e016fd5f996b Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Tue, 4 Feb 2025 10:48:17 +0200 Subject: [PATCH] `DeprecationInfoAction` refactoring (#121181) This refactoring was motivated by the following issues with the current state of the code: - The `TransformDeprecationChecker` is listed as plugin checker, but later we remove is from the `plugin_settings` and add it to the `cluster_settings`. This made me consider that the checker might be dealing with transform deprecation warnings but if they are listed under the `cliuster_settings`, it fits better to be part of `ClusterDeprecationChecker`. - The `DeprecationInfo` is a data class, but it has a method `from` which constructs an `DeprecationInfo.Response` instance. However, this is not a simple factory class but it actually runs all the checks and it also tries to assert that it is not executed on a transport thread. Considering this, I thought it might fit better to the `TransportDeprecationInfoAction`, this way all the logic is in one place and all the checkers are wired and used in the same class. - Constructing the node settings deprecation issues requires to merge the deprecation warnings of the individual nodes. We considered bringing together the execution of the remote request and the construction of the response in a new class called `NodeDeprecationChecker` that resembles the patterns of the other Checker classes. - Reinstated the `PLUGIN_CHECKERS` even if we have only one check, so other developers can easier add their plugin checks. - Finally, we noticed that the way we synthesise the remote requests is difficult to read and maintain because each call is nested under the previous one. We propose in this PR a different pattern that uses the `RefCountingListener` to combine the different remote calls and store their results in a container class named `PrecomputedData` - **Bonus**: Removed the `LegacyIndexTemplateDeprecationChecker.java` which was not used. --- .../upgrades/SourceModeRollingUpgradeIT.java | 21 +- .../src/main/java/module-info.java | 1 + .../ClusterDeprecationChecker.java | 56 +++ .../DataStreamDeprecationChecker.java | 23 +- .../xpack/deprecation/Deprecation.java | 2 +- .../xpack/deprecation/DeprecationChecks.java | 107 ----- .../deprecation/DeprecationInfoAction.java | 240 ----------- .../IlmPolicyDeprecationChecker.java | 32 +- .../deprecation/IndexDeprecationChecker.java | 110 +++-- ...LegacyIndexTemplateDeprecationChecker.java | 77 ---- .../deprecation/NodeDeprecationChecker.java | 138 +++++++ .../deprecation/NodeDeprecationChecks.java | 68 +++- .../ResourceDeprecationChecker.java | 8 +- .../TemplateDeprecationChecker.java | 44 +- .../TransformDeprecationChecker.java | 48 --- .../TransportDeprecationInfoAction.java | 343 ++++++++++++---- .../TransportNodeDeprecationCheckAction.java | 17 +- .../DataStreamDeprecationCheckerTests.java | 8 +- .../deprecation/DeprecationChecksTests.java | 48 --- .../DeprecationInfoActionResponseTests.java | 377 +----------------- .../IlmPolicyDeprecationCheckerTests.java | 4 +- .../IndexDeprecationCheckerTests.java | 94 +++-- .../NodeDeprecationCheckerTests.java | 74 ++++ .../NodeDeprecationChecksTests.java | 76 ++-- .../TemplateDeprecationCheckerTests.java | 8 +- .../TransportDeprecationInfoActionTests.java | 295 ++++++++++++++ ...nsportNodeDeprecationCheckActionTests.java | 17 +- 27 files changed, 1185 insertions(+), 1151 deletions(-) create mode 100644 x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecker.java delete mode 100644 x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java delete mode 100644 x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/LegacyIndexTemplateDeprecationChecker.java create mode 100644 x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecker.java delete mode 100644 x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransformDeprecationChecker.java delete mode 100644 x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationChecksTests.java create mode 100644 x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationCheckerTests.java diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SourceModeRollingUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SourceModeRollingUpgradeIT.java index 98572de6618ee..8bf870edc5ec8 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SourceModeRollingUpgradeIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SourceModeRollingUpgradeIT.java @@ -18,7 +18,6 @@ import java.util.List; import java.util.Map; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class SourceModeRollingUpgradeIT extends AbstractRollingUpgradeTestCase { @@ -83,20 +82,10 @@ public void testConfigureStoredSourceWhenIndexIsCreatedLegacy() throws IOExcepti private void assertDeprecationWarningForTemplate(String templateName) throws IOException { var request = new Request("GET", "/_migration/deprecations"); var response = entityAsMap(client().performRequest(request)); - if (response.containsKey("templates")) { - // Check the newer version of the deprecation API that contains the templates section - Map issuesByTemplate = (Map) response.get("templates"); - assertThat(issuesByTemplate.containsKey(templateName), equalTo(true)); - var templateIssues = (List) issuesByTemplate.get(templateName); - assertThat(((Map) templateIssues.getFirst()).get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING)); - } else { - // Bwc version with 8.18 until https://github.com/elastic/elasticsearch/pull/120505/ gets backported, clean up after backport - var nodeSettings = (Map) ((List) response.get("node_settings")).getFirst(); - assertThat(nodeSettings.get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING)); - assertThat( - (String) nodeSettings.get("details"), - containsString(SourceFieldMapper.DEPRECATION_WARNING + " Affected component templates: [" + templateName + "]") - ); - } + assertThat(response.containsKey("templates"), equalTo(true)); + Map issuesByTemplate = (Map) response.get("templates"); + assertThat(issuesByTemplate.containsKey(templateName), equalTo(true)); + var templateIssues = (List) issuesByTemplate.get(templateName); + assertThat(((Map) templateIssues.getFirst()).get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING)); } } diff --git a/x-pack/plugin/deprecation/src/main/java/module-info.java b/x-pack/plugin/deprecation/src/main/java/module-info.java index f9a86839ad6f2..4c46205df4f0c 100644 --- a/x-pack/plugin/deprecation/src/main/java/module-info.java +++ b/x-pack/plugin/deprecation/src/main/java/module-info.java @@ -13,6 +13,7 @@ requires org.apache.logging.log4j; requires org.apache.logging.log4j.core; requires log4j2.ecs.layout; + requires org.apache.lucene.core; exports org.elasticsearch.xpack.deprecation to org.elasticsearch.server; exports org.elasticsearch.xpack.deprecation.logging to org.elasticsearch.server; diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecker.java new file mode 100644 index 0000000000000..cc21f0b2cd711 --- /dev/null +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecker.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.deprecation; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.common.TriConsumer; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; +import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Cluster-specific deprecation checks, this is used to populate the {@code cluster_settings} field + */ +public class ClusterDeprecationChecker { + + private static final Logger logger = LogManager.getLogger(ClusterDeprecationChecker.class); + private final List, List>> CHECKS = List.of( + this::checkTransformSettings + ); + private final NamedXContentRegistry xContentRegistry; + + ClusterDeprecationChecker(NamedXContentRegistry xContentRegistry) { + this.xContentRegistry = xContentRegistry; + } + + public List check(ClusterState clusterState, List transformConfigs) { + List allIssues = new ArrayList<>(); + CHECKS.forEach(check -> check.apply(clusterState, transformConfigs, allIssues)); + return allIssues; + } + + private void checkTransformSettings( + ClusterState clusterState, + List transformConfigs, + List allIssues + ) { + for (var config : transformConfigs) { + try { + allIssues.addAll(config.checkForDeprecations(xContentRegistry)); + } catch (IOException e) { + logger.warn("failed to check transformation settings for '" + config.getId() + "'", e); + } + } + } +} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java index 4844722ddcdd4..db8a3dc205e02 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java @@ -18,13 +18,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.BiFunction; import java.util.stream.Collectors; import static java.util.Map.entry; import static java.util.Map.ofEntries; -import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.filterChecks; /** * Checks the data streams for deprecation warnings. @@ -44,10 +44,24 @@ public DataStreamDeprecationChecker(IndexNameExpressionResolver indexNameExpress /** * @param clusterState The cluster state provided for the checker + * @param request not used yet in these checks + * @param precomputedData not used yet in these checks * @return the name of the data streams that have violated the checks with their respective warnings. */ @Override - public Map> check(ClusterState clusterState, DeprecationInfoAction.Request request) { + public Map> check( + ClusterState clusterState, + DeprecationInfoAction.Request request, + TransportDeprecationInfoAction.PrecomputedData precomputedData + ) { + return check(clusterState); + } + + /** + * @param clusterState The cluster state provided for the checker + * @return the name of the data streams that have violated the checks with their respective warnings. + */ + public Map> check(ClusterState clusterState) { List dataStreamNames = indexNameExpressionResolver.dataStreamNames( clusterState, IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN @@ -58,7 +72,10 @@ public Map> check(ClusterState clusterState, Depr Map> dataStreamIssues = new HashMap<>(); for (String dataStreamName : dataStreamNames) { DataStream dataStream = clusterState.metadata().dataStreams().get(dataStreamName); - List issuesForSingleDataStream = filterChecks(DATA_STREAM_CHECKS, c -> c.apply(dataStream, clusterState)); + List issuesForSingleDataStream = DATA_STREAM_CHECKS.stream() + .map(c -> c.apply(dataStream, clusterState)) + .filter(Objects::nonNull) + .toList(); if (issuesForSingleDataStream.isEmpty() == false) { dataStreamIssues.put(dataStreamName, issuesForSingleDataStream); } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java index 85b7c89e7cb85..2c8b95e378375 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java @@ -33,7 +33,7 @@ import java.util.function.Predicate; import java.util.function.Supplier; -import static org.elasticsearch.xpack.deprecation.DeprecationChecks.SKIP_DEPRECATIONS_SETTING; +import static org.elasticsearch.xpack.deprecation.TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING; import static org.elasticsearch.xpack.deprecation.logging.DeprecationIndexingComponent.DEPRECATION_INDEXING_FLUSH_INTERVAL; /** diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java deleted file mode 100644 index 039a75f51f030..0000000000000 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.deprecation; - -import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; - -import java.util.List; -import java.util.Objects; -import java.util.function.Function; -import java.util.stream.Collectors; - -/** - * Class containing all the cluster, node, and index deprecation checks that will be served - * by the {@link DeprecationInfoAction}. - */ -public class DeprecationChecks { - - public static final Setting> SKIP_DEPRECATIONS_SETTING = Setting.stringListSetting( - "deprecation.skip_deprecated_settings", - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - private DeprecationChecks() {} - - static List> CLUSTER_SETTINGS_CHECKS = List.of(); - - static final List< - NodeDeprecationCheck> NODE_SETTINGS_CHECKS = List - .of( - NodeDeprecationChecks::checkMultipleDataPaths, - NodeDeprecationChecks::checkDataPathsList, - NodeDeprecationChecks::checkSharedDataPathSetting, - NodeDeprecationChecks::checkReservedPrefixedRealmNames, - NodeDeprecationChecks::checkExporterUseIngestPipelineSettings, - NodeDeprecationChecks::checkExporterPipelineMasterTimeoutSetting, - NodeDeprecationChecks::checkExporterCreateLegacyTemplateSetting, - NodeDeprecationChecks::checkMonitoringSettingHistoryDuration, - NodeDeprecationChecks::checkMonitoringSettingHistoryDuration, - NodeDeprecationChecks::checkMonitoringSettingCollectIndexRecovery, - NodeDeprecationChecks::checkMonitoringSettingCollectIndices, - NodeDeprecationChecks::checkMonitoringSettingCollectCcrTimeout, - NodeDeprecationChecks::checkMonitoringSettingCollectEnrichStatsTimeout, - NodeDeprecationChecks::checkMonitoringSettingCollectIndexRecoveryStatsTimeout, - NodeDeprecationChecks::checkMonitoringSettingCollectIndexStatsTimeout, - NodeDeprecationChecks::checkMonitoringSettingCollectMlJobStatsTimeout, - NodeDeprecationChecks::checkMonitoringSettingCollectNodeStatsTimeout, - NodeDeprecationChecks::checkMonitoringSettingCollectClusterStatsTimeout, - NodeDeprecationChecks::checkMonitoringSettingExportersHost, - NodeDeprecationChecks::checkMonitoringSettingExportersBulkTimeout, - NodeDeprecationChecks::checkMonitoringSettingExportersConnectionTimeout, - NodeDeprecationChecks::checkMonitoringSettingExportersConnectionReadTimeout, - NodeDeprecationChecks::checkMonitoringSettingExportersAuthUsername, - NodeDeprecationChecks::checkMonitoringSettingExportersAuthPass, - NodeDeprecationChecks::checkMonitoringSettingExportersSSL, - NodeDeprecationChecks::checkMonitoringSettingExportersProxyBase, - NodeDeprecationChecks::checkMonitoringSettingExportersSniffEnabled, - NodeDeprecationChecks::checkMonitoringSettingExportersHeaders, - NodeDeprecationChecks::checkMonitoringSettingExportersTemplateTimeout, - NodeDeprecationChecks::checkMonitoringSettingExportersMasterTimeout, - NodeDeprecationChecks::checkMonitoringSettingExportersEnabled, - NodeDeprecationChecks::checkMonitoringSettingExportersType, - NodeDeprecationChecks::checkMonitoringSettingExportersAlertsEnabled, - NodeDeprecationChecks::checkMonitoringSettingExportersAlertsBlacklist, - NodeDeprecationChecks::checkMonitoringSettingExportersIndexNameTimeFormat, - NodeDeprecationChecks::checkMonitoringSettingDecommissionAlerts, - NodeDeprecationChecks::checkMonitoringSettingEsCollectionEnabled, - NodeDeprecationChecks::checkMonitoringSettingCollectionEnabled, - NodeDeprecationChecks::checkMonitoringSettingCollectionInterval, - NodeDeprecationChecks::checkScriptContextCache, - NodeDeprecationChecks::checkScriptContextCompilationsRateLimitSetting, - NodeDeprecationChecks::checkScriptContextCacheSizeSetting, - NodeDeprecationChecks::checkScriptContextCacheExpirationSetting, - NodeDeprecationChecks::checkEnforceDefaultTierPreferenceSetting, - NodeDeprecationChecks::checkLifecyleStepMasterTimeoutSetting, - NodeDeprecationChecks::checkEqlEnabledSetting, - NodeDeprecationChecks::checkNodeAttrData, - NodeDeprecationChecks::checkWatcherBulkConcurrentRequestsSetting, - NodeDeprecationChecks::checkTracingApmSettings - ); - - /** - * helper utility function to reduce repeat of running a specific {@link List} of checks. - * - * @param checks The functional checks to execute using the mapper function - * @param mapper The function that executes the lambda check with the appropriate arguments - * @param The signature of the check (BiFunction, Function, including the appropriate arguments) - * @return The list of {@link DeprecationIssue} that were found in the cluster - */ - static List filterChecks(List checks, Function mapper) { - return checks.stream().map(mapper).filter(Objects::nonNull).collect(Collectors.toList()); - } - - @FunctionalInterface - public interface NodeDeprecationCheck { - R apply(A first, B second, C third, D fourth); - } -} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java index 62d2a231face4..1fceb917ece53 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java @@ -12,41 +12,25 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.IndicesRequest; -import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeReadRequest; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.ComponentTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.regex.Regex; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.core.Tuple; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.transport.Transports; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; import static org.elasticsearch.action.ValidateActions.addValidationError; @@ -59,93 +43,6 @@ private DeprecationInfoAction() { super(NAME); } - /** - * helper utility function to reduce repeat of running a specific {@link Set} of checks. - * - * @param checks The functional checks to execute using the mapper function - * @param mapper The function that executes the lambda check with the appropriate arguments - * @param The signature of the check (BiFunction, Function, including the appropriate arguments) - * @return The list of {@link DeprecationIssue} that were found in the cluster - */ - public static List filterChecks(List checks, Function mapper) { - return checks.stream().map(mapper).filter(Objects::nonNull).collect(Collectors.toList()); - } - - /** - * This method rolls up DeprecationIssues that are identical but on different nodes. It also roles up DeprecationIssues that are - * identical (and on different nodes) except that they differ in the removable settings listed in their meta object. We roll these up - * by taking the intersection of all removable settings in otherwise identical DeprecationIssues. That way we don't claim that a - * setting can be automatically removed if any node has it in its elasticsearch.yml. - * @param response - * @return - */ - private static List mergeNodeIssues(NodesDeprecationCheckResponse response) { - // A collection whose values are lists of DeprecationIssues that differ only by meta values (if that): - Collection>> issuesToMerge = getDeprecationIssuesThatDifferOnlyByMeta(response.getNodes()); - // A map of DeprecationIssues (containing only the intersection of removable settings) to the nodes they are seen on - Map> issueToListOfNodesMap = getMergedIssuesToNodesMap(issuesToMerge); - - return issueToListOfNodesMap.entrySet().stream().map(entry -> { - DeprecationIssue issue = entry.getKey(); - String details = issue.getDetails() != null ? issue.getDetails() + " " : ""; - return new DeprecationIssue( - issue.getLevel(), - issue.getMessage(), - issue.getUrl(), - details + "(nodes impacted: " + entry.getValue() + ")", - issue.isResolveDuringRollingUpgrade(), - issue.getMeta() - ); - }).collect(Collectors.toList()); - } - - /* - * This method pulls all the DeprecationIssues from the given nodeResponses, and buckets them into lists of DeprecationIssues that - * differ at most by meta values (if that). The returned tuples also contain the node name the deprecation issue was found on. If all - * nodes in the cluster were configured identically then all tuples in a list will differ only by the node name. - */ - private static Collection>> getDeprecationIssuesThatDifferOnlyByMeta( - List nodeResponses - ) { - Map>> issuesToMerge = new HashMap<>(); - for (NodesDeprecationCheckAction.NodeResponse resp : nodeResponses) { - for (DeprecationIssue issue : resp.getDeprecationIssues()) { - issuesToMerge.computeIfAbsent( - new DeprecationIssue( - issue.getLevel(), - issue.getMessage(), - issue.getUrl(), - issue.getDetails(), - issue.isResolveDuringRollingUpgrade(), - null // Intentionally removing meta from the key so that it's not taken into account for equality - ), - (key) -> new ArrayList<>() - ).add(new Tuple<>(issue, resp.getNode().getName())); - } - } - return issuesToMerge.values(); - } - - /* - * At this point we have one DeprecationIssue per node for a given deprecation. This method rolls them up into a single DeprecationIssue - * with a list of nodes that they appear on. If two DeprecationIssues on two different nodes differ only by the set of removable - * settings (i.e. they have different elasticsearch.yml configurations) then this method takes the intersection of those settings when - * it rolls them up. - */ - private static Map> getMergedIssuesToNodesMap( - Collection>> issuesToMerge - ) { - Map> issueToListOfNodesMap = new HashMap<>(); - for (List> similarIssues : issuesToMerge) { - DeprecationIssue leastCommonDenominator = DeprecationIssue.getIntersectionOfRemovableSettings( - similarIssues.stream().map(Tuple::v1).toList() - ); - issueToListOfNodesMap.computeIfAbsent(leastCommonDenominator, (key) -> new ArrayList<>()) - .addAll(similarIssues.stream().map(Tuple::v2).toList()); - } - return issueToListOfNodesMap; - } - public static class Response extends ActionResponse implements ToXContentObject { static final Set RESERVED_NAMES = Set.of( "cluster_settings", @@ -289,143 +186,6 @@ public int hashCode() { return Objects.hash(clusterSettingsIssues, nodeSettingsIssues, resourceDeprecationIssues, pluginSettingsIssues); } - /** - * This is the function that does the bulk of the logic of taking the appropriate ES dependencies - * like {@link NodeInfo}, {@link ClusterState}. Alongside these objects and the list of deprecation checks, - * this function will run through all the checks and build out the final list of issues that exist in the - * cluster. - * - * @param state The cluster state - * @param indexNameExpressionResolver Used to resolve indices into their concrete names - * @param request The originating request containing the index expressions to evaluate - * @param nodeDeprecationResponse The response containing the deprecation issues found on each node - * @param clusterSettingsChecks The list of cluster-level checks - * @param pluginSettingIssues this map gets modified to move transform deprecation issues into cluster_settings - * @param skipTheseDeprecatedSettings the settings that will be removed from cluster metadata and the index metadata of all the - * indexes specified by indexNames - * @param resourceDeprecationCheckers these are checkers that take as input the cluster state and return a map from resource type - * to issues grouped by the resource name. - * @return The list of deprecation issues found in the cluster - */ - public static DeprecationInfoAction.Response from( - ClusterState state, - IndexNameExpressionResolver indexNameExpressionResolver, - Request request, - NodesDeprecationCheckResponse nodeDeprecationResponse, - List> clusterSettingsChecks, - Map> pluginSettingIssues, - List skipTheseDeprecatedSettings, - List resourceDeprecationCheckers - ) { - assert Transports.assertNotTransportThread("walking mappings in indexSettingsChecks is expensive"); - // Allow system index access here to prevent deprecation warnings when we call this API - String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames(state, request); - ClusterState stateWithSkippedSettingsRemoved = removeSkippedSettings(state, concreteIndexNames, skipTheseDeprecatedSettings); - List clusterSettingsIssues = filterChecks( - clusterSettingsChecks, - (c) -> c.apply(stateWithSkippedSettingsRemoved) - ); - List nodeSettingsIssues = mergeNodeIssues(nodeDeprecationResponse); - - Map>> resourceDeprecationIssues = new HashMap<>(); - for (ResourceDeprecationChecker resourceDeprecationChecker : resourceDeprecationCheckers) { - Map> issues = resourceDeprecationChecker.check(stateWithSkippedSettingsRemoved, request); - if (issues.isEmpty() == false) { - resourceDeprecationIssues.put(resourceDeprecationChecker.getName(), issues); - } - } - - // WORKAROUND: move transform deprecation issues into cluster_settings - List transformDeprecations = pluginSettingIssues.remove( - TransformDeprecationChecker.TRANSFORM_DEPRECATION_KEY - ); - if (transformDeprecations != null) { - clusterSettingsIssues.addAll(transformDeprecations); - } - - return new DeprecationInfoAction.Response( - clusterSettingsIssues, - nodeSettingsIssues, - resourceDeprecationIssues, - pluginSettingIssues - ); - } - } - - /** - * Removes the skipped settings from the selected indices and the component and index templates. - * @param state The cluster state to modify - * @param indexNames The names of the indexes whose settings need to be filtered - * @param skipTheseDeprecatedSettings The settings that will be removed from cluster metadata and the index metadata of all the - * indexes specified by indexNames - * @return A modified cluster state with the given settings removed - */ - private static ClusterState removeSkippedSettings(ClusterState state, String[] indexNames, List skipTheseDeprecatedSettings) { - // Short-circuit, no need to reconstruct the cluster state if there are no settings to remove - if (skipTheseDeprecatedSettings == null || skipTheseDeprecatedSettings.isEmpty()) { - return state; - } - ClusterState.Builder clusterStateBuilder = new ClusterState.Builder(state); - Metadata.Builder metadataBuilder = Metadata.builder(state.metadata()); - metadataBuilder.transientSettings( - metadataBuilder.transientSettings().filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false) - ); - metadataBuilder.persistentSettings( - metadataBuilder.persistentSettings().filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false) - ); - Map indicesBuilder = new HashMap<>(state.getMetadata().indices()); - for (String indexName : indexNames) { - IndexMetadata indexMetadata = state.getMetadata().index(indexName); - IndexMetadata.Builder filteredIndexMetadataBuilder = new IndexMetadata.Builder(indexMetadata); - Settings filteredSettings = indexMetadata.getSettings() - .filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false); - filteredIndexMetadataBuilder.settings(filteredSettings); - indicesBuilder.put(indexName, filteredIndexMetadataBuilder.build()); - } - metadataBuilder.componentTemplates(state.metadata().componentTemplates().entrySet().stream().map(entry -> { - String templateName = entry.getKey(); - ComponentTemplate componentTemplate = entry.getValue(); - Template template = componentTemplate.template(); - if (template.settings() == null || template.settings().isEmpty()) { - return Tuple.tuple(templateName, componentTemplate); - } - return Tuple.tuple( - templateName, - new ComponentTemplate( - Template.builder(template) - .settings(template.settings().filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false)) - .build(), - componentTemplate.version(), - componentTemplate.metadata(), - componentTemplate.deprecated() - ) - ); - }).collect(Collectors.toMap(Tuple::v1, Tuple::v2))); - metadataBuilder.indexTemplates(state.metadata().templatesV2().entrySet().stream().map(entry -> { - String templateName = entry.getKey(); - ComposableIndexTemplate indexTemplate = entry.getValue(); - Template template = indexTemplate.template(); - if (template == null || template.settings() == null || template.settings().isEmpty()) { - return Tuple.tuple(templateName, indexTemplate); - } - return Tuple.tuple( - templateName, - indexTemplate.toBuilder() - .template( - Template.builder(indexTemplate.template()) - .settings( - indexTemplate.template() - .settings() - .filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false) - ) - ) - .build() - ); - }).collect(Collectors.toMap(Tuple::v1, Tuple::v2))); - - metadataBuilder.indices(indicesBuilder); - clusterStateBuilder.metadata(metadataBuilder); - return clusterStateBuilder.build(); } public static class Request extends MasterNodeReadRequest implements IndicesRequest.Replaceable { diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationChecker.java index 6d7f860f645f1..f96fae6343b9f 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationChecker.java @@ -19,9 +19,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.function.Function; -import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.filterChecks; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_COMMON_DETAIL; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_HELP_URL; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_MESSAGE; @@ -33,17 +33,28 @@ public class IlmPolicyDeprecationChecker implements ResourceDeprecationChecker { public static final String NAME = "ilm_policies"; - private static final List> CHECKS = List.of( - IlmPolicyDeprecationChecker::checkLegacyTiers, - IlmPolicyDeprecationChecker::checkFrozenAction - ); + private final List> checks = List.of(this::checkLegacyTiers, this::checkFrozenAction); /** * @param clusterState The cluster state provided for the checker + * @param request not used yet in these checks + * @param precomputedData not used yet in these checks * @return the name of the data streams that have violated the checks with their respective warnings. */ @Override - public Map> check(ClusterState clusterState, DeprecationInfoAction.Request request) { + public Map> check( + ClusterState clusterState, + DeprecationInfoAction.Request request, + TransportDeprecationInfoAction.PrecomputedData precomputedData + ) { + return check(clusterState); + } + + /** + * @param clusterState The cluster state provided for the checker + * @return the name of the data streams that have violated the checks with their respective warnings. + */ + Map> check(ClusterState clusterState) { IndexLifecycleMetadata lifecycleMetadata = clusterState.metadata().custom(IndexLifecycleMetadata.TYPE); if (lifecycleMetadata == null || lifecycleMetadata.getPolicyMetadatas().isEmpty()) { return Map.of(); @@ -53,7 +64,10 @@ public Map> check(ClusterState clusterState, Depr String name = entry.getKey(); LifecyclePolicyMetadata policyMetadata = entry.getValue(); - List issuesForSinglePolicy = filterChecks(CHECKS, c -> c.apply(policyMetadata.getPolicy())); + List issuesForSinglePolicy = checks.stream() + .map(c -> c.apply(policyMetadata.getPolicy())) + .filter(Objects::nonNull) + .toList(); if (issuesForSinglePolicy.isEmpty() == false) { issues.put(name, issuesForSinglePolicy); } @@ -61,7 +75,7 @@ public Map> check(ClusterState clusterState, Depr return issues.isEmpty() ? Map.of() : issues; } - static DeprecationIssue checkLegacyTiers(LifecyclePolicy policy) { + private DeprecationIssue checkLegacyTiers(LifecyclePolicy policy) { for (Phase phase : policy.getPhases().values()) { AllocateAction allocateAction = (AllocateAction) phase.getActions().get(AllocateAction.NAME); if (allocateAction != null) { @@ -82,7 +96,7 @@ static DeprecationIssue checkLegacyTiers(LifecyclePolicy policy) { return null; } - static DeprecationIssue checkFrozenAction(LifecyclePolicy policy) { + private DeprecationIssue checkFrozenAction(LifecyclePolicy policy) { for (Phase phase : policy.getPhases().values()) { if (phase.getActions().containsKey(FreezeAction.NAME)) { return new DeprecationIssue( diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java index 778e4d176ca0f..844988c7a4896 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java @@ -10,6 +10,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.LegacyFormatNames; import org.elasticsearch.index.IndexModule; @@ -17,17 +18,19 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.xpack.core.deprecation.DeprecatedIndexPredicate; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; +import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Function; +import java.util.stream.Collectors; -import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.filterChecks; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_COMMON_DETAIL; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_HELP_URL; @@ -39,20 +42,35 @@ public class IndexDeprecationChecker implements ResourceDeprecationChecker { public static final String NAME = "index_settings"; private final IndexNameExpressionResolver indexNameExpressionResolver; - private final Map> indexToTransformIds; + private final List>, DeprecationIssue>> checks = List.of( + this::oldIndicesCheck, + this::ignoredOldIndicesCheck, + this::translogRetentionSettingCheck, + this::checkIndexDataPath, + this::storeTypeSettingCheck, + this::deprecatedCamelCasePattern, + this::legacyRoutingSettingCheck + ); - public IndexDeprecationChecker(IndexNameExpressionResolver indexNameExpressionResolver, Map> indexToTransformIds) { + public IndexDeprecationChecker(IndexNameExpressionResolver indexNameExpressionResolver) { this.indexNameExpressionResolver = indexNameExpressionResolver; - this.indexToTransformIds = indexToTransformIds; } @Override - public Map> check(ClusterState clusterState, DeprecationInfoAction.Request request) { + public Map> check( + ClusterState clusterState, + DeprecationInfoAction.Request request, + TransportDeprecationInfoAction.PrecomputedData precomputedData + ) { Map> indexSettingsIssues = new HashMap<>(); String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames(clusterState, request); + Map> indexToTransformIds = indexToTransformIds(precomputedData.transformConfigs()); for (String concreteIndex : concreteIndexNames) { IndexMetadata indexMetadata = clusterState.getMetadata().index(concreteIndex); - List singleIndexIssues = filterChecks(indexSettingsChecks(), c -> c.apply(indexMetadata, clusterState)); + List singleIndexIssues = checks.stream() + .map(c -> c.apply(indexMetadata, clusterState, indexToTransformIds)) + .filter(Objects::nonNull) + .toList(); if (singleIndexIssues.isEmpty() == false) { indexSettingsIssues.put(concreteIndex, singleIndexIssues); } @@ -63,24 +81,16 @@ public Map> check(ClusterState clusterState, Depr return indexSettingsIssues; } - private List> indexSettingsChecks() { - return List.of( - this::oldIndicesCheck, - this::ignoredOldIndicesCheck, - IndexDeprecationChecker::translogRetentionSettingCheck, - IndexDeprecationChecker::checkIndexDataPath, - IndexDeprecationChecker::storeTypeSettingCheck, - IndexDeprecationChecker::deprecatedCamelCasePattern, - IndexDeprecationChecker::legacyRoutingSettingCheck - ); - } - @Override public String getName() { return NAME; } - private DeprecationIssue oldIndicesCheck(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue oldIndicesCheck( + IndexMetadata indexMetadata, + ClusterState clusterState, + Map> indexToTransformIds + ) { // TODO: this check needs to be revised. It's trivially true right now. IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks @@ -91,13 +101,13 @@ private DeprecationIssue oldIndicesCheck(IndexMetadata indexMetadata, ClusterSta "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html", "This index has version: " + currentCompatibilityVersion.toReleaseVersion(), false, - meta(indexMetadata) + meta(indexMetadata, indexToTransformIds) ); } return null; } - private Map meta(IndexMetadata indexMetadata) { + private Map meta(IndexMetadata indexMetadata, Map> indexToTransformIds) { var transforms = indexToTransformIds.getOrDefault(indexMetadata.getIndex().getName(), List.of()); if (transforms.isEmpty()) { return Map.of("reindex_required", true); @@ -106,7 +116,11 @@ private Map meta(IndexMetadata indexMetadata) { } } - private DeprecationIssue ignoredOldIndicesCheck(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue ignoredOldIndicesCheck( + IndexMetadata indexMetadata, + ClusterState clusterState, + Map> indexToTransformIds + ) { IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, true) && isNotDataStreamIndex(indexMetadata, clusterState)) { @@ -118,17 +132,21 @@ private DeprecationIssue ignoredOldIndicesCheck(IndexMetadata indexMetadata, Clu + currentCompatibilityVersion.toReleaseVersion() + " and will be supported as read-only in 9.0", false, - meta(indexMetadata) + meta(indexMetadata, indexToTransformIds) ); } return null; } - private static boolean isNotDataStreamIndex(IndexMetadata indexMetadata, ClusterState clusterState) { + private boolean isNotDataStreamIndex(IndexMetadata indexMetadata, ClusterState clusterState) { return clusterState.metadata().findDataStreams(indexMetadata.getIndex().getName()).isEmpty(); } - private static DeprecationIssue translogRetentionSettingCheck(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue translogRetentionSettingCheck( + IndexMetadata indexMetadata, + ClusterState clusterState, + Map> ignored + ) { final boolean softDeletesEnabled = IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexMetadata.getSettings()); if (softDeletesEnabled) { if (IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.exists(indexMetadata.getSettings()) @@ -155,7 +173,7 @@ private static DeprecationIssue translogRetentionSettingCheck(IndexMetadata inde return null; } - private static DeprecationIssue checkIndexDataPath(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue checkIndexDataPath(IndexMetadata indexMetadata, ClusterState clusterState, Map> ignored) { if (IndexMetadata.INDEX_DATA_PATH_SETTING.exists(indexMetadata.getSettings())) { final String message = String.format( Locale.ROOT, @@ -170,7 +188,11 @@ private static DeprecationIssue checkIndexDataPath(IndexMetadata indexMetadata, return null; } - private static DeprecationIssue storeTypeSettingCheck(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue storeTypeSettingCheck( + IndexMetadata indexMetadata, + ClusterState clusterState, + Map> ignored + ) { final String storeType = IndexModule.INDEX_STORE_TYPE_SETTING.get(indexMetadata.getSettings()); if (IndexModule.Type.SIMPLEFS.match(storeType)) { return new DeprecationIssue( @@ -187,7 +209,11 @@ private static DeprecationIssue storeTypeSettingCheck(IndexMetadata indexMetadat return null; } - private static DeprecationIssue legacyRoutingSettingCheck(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue legacyRoutingSettingCheck( + IndexMetadata indexMetadata, + ClusterState clusterState, + Map> ignored + ) { List deprecatedSettings = LegacyTiersDetection.getDeprecatedFilteredAllocationSettings(indexMetadata.getSettings()); if (deprecatedSettings.isEmpty()) { return null; @@ -203,7 +229,7 @@ private static DeprecationIssue legacyRoutingSettingCheck(IndexMetadata indexMet ); } - private static void fieldLevelMappingIssue(IndexMetadata indexMetadata, BiConsumer> checker) { + private void fieldLevelMappingIssue(IndexMetadata indexMetadata, BiConsumer> checker) { if (indexMetadata.mapping() != null) { Map sourceAsMap = indexMetadata.mapping().sourceAsMap(); checker.accept(indexMetadata.mapping(), sourceAsMap); @@ -221,7 +247,7 @@ private static void fieldLevelMappingIssue(IndexMetadata indexMetadata, BiConsum * @return a list of issues found in fields */ @SuppressWarnings("unchecked") - private static List findInPropertiesRecursively( + private List findInPropertiesRecursively( String type, Map parentMap, Function, Boolean> predicate, @@ -275,7 +301,11 @@ private static List findInPropertiesRecursively( return issues; } - private static DeprecationIssue deprecatedCamelCasePattern(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue deprecatedCamelCasePattern( + IndexMetadata indexMetadata, + ClusterState clusterState, + Map> ignored + ) { List fields = new ArrayList<>(); fieldLevelMappingIssue( indexMetadata, @@ -283,8 +313,8 @@ private static DeprecationIssue deprecatedCamelCasePattern(IndexMetadata indexMe findInPropertiesRecursively( mappingMetadata.type(), sourceAsMap, - IndexDeprecationChecker::isDateFieldWithCamelCasePattern, - IndexDeprecationChecker::changeFormatToSnakeCase, + this::isDateFieldWithCamelCasePattern, + this::changeFormatToSnakeCase, "", "" ) @@ -305,7 +335,7 @@ private static DeprecationIssue deprecatedCamelCasePattern(IndexMetadata indexMe return null; } - private static boolean isDateFieldWithCamelCasePattern(Map property) { + private boolean isDateFieldWithCamelCasePattern(Map property) { if ("date".equals(property.get("type")) && property.containsKey("format")) { String[] patterns = DateFormatter.splitCombinedPatterns((String) property.get("format")); for (String pattern : patterns) { @@ -316,7 +346,7 @@ private static boolean isDateFieldWithCamelCasePattern(Map property) { return false; } - private static String changeFormatToSnakeCase(String type, Map.Entry entry) { + private String changeFormatToSnakeCase(String type, Map.Entry entry) { Map value = (Map) entry.getValue(); final String formatFieldValue = (String) value.get("format"); String[] patterns = DateFormatter.splitCombinedPatterns(formatFieldValue); @@ -332,4 +362,14 @@ private static String changeFormatToSnakeCase(String type, Map.Entry entry sb.deleteCharAt(sb.length() - 1); return sb.toString(); } + + private Map> indexToTransformIds(List transformConfigs) { + return transformConfigs.stream() + .collect( + Collectors.groupingBy( + config -> config.getDestination().getIndex(), + Collectors.mapping(TransformConfig::getId, Collectors.toList()) + ) + ); + } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/LegacyIndexTemplateDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/LegacyIndexTemplateDeprecationChecker.java deleted file mode 100644 index f7aba6491dfd2..0000000000000 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/LegacyIndexTemplateDeprecationChecker.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.deprecation; - -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; -import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Function; - -import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.filterChecks; -import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_COMMON_DETAIL; -import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_HELP_URL; -import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_MESSAGE; - -/** - * Checks the legacy index templates for deprecation warnings. - */ -public class LegacyIndexTemplateDeprecationChecker implements ResourceDeprecationChecker { - - public static final String NAME = "legacy_templates"; - private static final List> CHECKS = List.of( - LegacyIndexTemplateDeprecationChecker::checkIndexTemplates - ); - - /** - * @param clusterState The cluster state provided for the checker - * @return the name of the data streams that have violated the checks with their respective warnings. - */ - @Override - public Map> check(ClusterState clusterState, DeprecationInfoAction.Request request) { - var templates = clusterState.metadata().templates().entrySet(); - if (templates.isEmpty()) { - return Map.of(); - } - Map> issues = new HashMap<>(); - for (Map.Entry entry : templates) { - String name = entry.getKey(); - IndexTemplateMetadata template = entry.getValue(); - - List issuesForSingleIndexTemplate = filterChecks(CHECKS, c -> c.apply(template)); - if (issuesForSingleIndexTemplate.isEmpty() == false) { - issues.put(name, issuesForSingleIndexTemplate); - } - } - return issues.isEmpty() ? Map.of() : issues; - } - - static DeprecationIssue checkIndexTemplates(IndexTemplateMetadata indexTemplateMetadata) { - List deprecatedSettings = LegacyTiersDetection.getDeprecatedFilteredAllocationSettings(indexTemplateMetadata.settings()); - if (deprecatedSettings.isEmpty()) { - return null; - } - return new DeprecationIssue( - DeprecationIssue.Level.WARNING, - DEPRECATION_MESSAGE, - DEPRECATION_HELP_URL, - "One or more of your legacy index templates is configured with 'index.routing.allocation.*.data' settings. " - + DEPRECATION_COMMON_DETAIL, - false, - DeprecationIssue.createMetaMapForRemovableSettings(deprecatedSettings) - ); - } - - @Override - public String getName() { - return NAME; - } -} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecker.java new file mode 100644 index 0000000000000..a2e9ed12a2298 --- /dev/null +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecker.java @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.deprecation; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.ThreadedActionListener; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Retrieves the individual node checks and reduces them to a list of deprecation warnings + */ +public class NodeDeprecationChecker { + + private static final Logger logger = LogManager.getLogger(NodeDeprecationChecker.class); + private final ThreadPool threadPool; + + public NodeDeprecationChecker(ThreadPool threadPool) { + this.threadPool = threadPool; + } + + public void check(Client client, ActionListener> listener) { + NodesDeprecationCheckRequest nodeDepReq = new NodesDeprecationCheckRequest("_all"); + ClientHelper.executeAsyncWithOrigin( + client, + ClientHelper.DEPRECATION_ORIGIN, + NodesDeprecationCheckAction.INSTANCE, + nodeDepReq, + new ThreadedActionListener<>(threadPool.generic(), listener.delegateFailureAndWrap((l, response) -> { + if (response.hasFailures()) { + List failedNodeIds = response.failures() + .stream() + .map(failure -> failure.nodeId() + ": " + failure.getMessage()) + .collect(Collectors.toList()); + logger.warn("nodes failed to run deprecation checks: {}", failedNodeIds); + for (FailedNodeException failure : response.failures()) { + logger.debug("node {} failed to run deprecation checks: {}", failure.nodeId(), failure); + } + } + l.onResponse(reduceToDeprecationIssues(response)); + })) + ); + } + + /** + * This method rolls up DeprecationIssues that are identical but on different nodes. It also rolls up DeprecationIssues that are + * identical (and on different nodes) except that they differ in the removable settings listed in their meta object. We roll these up + * by taking the intersection of all removable settings in otherwise identical DeprecationIssues. That way we don't claim that a + * setting can be automatically removed if any node has it in its elasticsearch.yml. + * @param response the response that contains the deprecation issues of single nodes + * @return a list of deprecation issues grouped accordingly. + */ + static List reduceToDeprecationIssues(NodesDeprecationCheckResponse response) { + // A collection whose values are lists of DeprecationIssues that differ only by meta values (if that): + Collection>> issuesToMerge = getDeprecationIssuesThatDifferOnlyByMeta(response.getNodes()); + // A map of DeprecationIssues (containing only the intersection of removable settings) to the nodes they are seen on + Map> issueToListOfNodesMap = getMergedIssuesToNodesMap(issuesToMerge); + + return issueToListOfNodesMap.entrySet().stream().map(entry -> { + DeprecationIssue issue = entry.getKey(); + String details = issue.getDetails() != null ? issue.getDetails() + " " : ""; + return new DeprecationIssue( + issue.getLevel(), + issue.getMessage(), + issue.getUrl(), + details + "(nodes impacted: " + entry.getValue() + ")", + issue.isResolveDuringRollingUpgrade(), + issue.getMeta() + ); + }).collect(Collectors.toList()); + } + + /* + * This method pulls all the DeprecationIssues from the given nodeResponses, and buckets them into lists of DeprecationIssues that + * differ at most by meta values (if that). The returned tuples also contain the node name the deprecation issue was found on. If all + * nodes in the cluster were configured identically then all tuples in a list will differ only by the node name. + */ + private static Collection>> getDeprecationIssuesThatDifferOnlyByMeta( + List nodeResponses + ) { + Map>> issuesToMerge = new HashMap<>(); + for (NodesDeprecationCheckAction.NodeResponse resp : nodeResponses) { + for (DeprecationIssue issue : resp.getDeprecationIssues()) { + issuesToMerge.computeIfAbsent( + new DeprecationIssue( + issue.getLevel(), + issue.getMessage(), + issue.getUrl(), + issue.getDetails(), + issue.isResolveDuringRollingUpgrade(), + null // Intentionally removing meta from the key so that it's not taken into account for equality + ), + (key) -> new ArrayList<>() + ).add(new Tuple<>(issue, resp.getNode().getName())); + } + } + return issuesToMerge.values(); + } + + /* + * At this point we have one DeprecationIssue per node for a given deprecation. This method rolls them up into a single DeprecationIssue + * with a list of nodes that they appear on. If two DeprecationIssues on two different nodes differ only by the set of removable + * settings (i.e. they have different elasticsearch.yml configurations) then this method takes the intersection of those settings when + * it rolls them up. + */ + private static Map> getMergedIssuesToNodesMap( + Collection>> issuesToMerge + ) { + Map> issueToListOfNodesMap = new HashMap<>(); + for (List> similarIssues : issuesToMerge) { + DeprecationIssue leastCommonDenominator = DeprecationIssue.getIntersectionOfRemovableSettings( + similarIssues.stream().map(Tuple::v1).toList() + ); + issueToListOfNodesMap.computeIfAbsent(leastCommonDenominator, (key) -> new ArrayList<>()) + .addAll(similarIssues.stream().map(Tuple::v2).toList()); + } + return issueToListOfNodesMap; + } + +} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java index b6fff5a82f0cd..2f476d111f4ba 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java @@ -41,6 +41,60 @@ public class NodeDeprecationChecks { + // Visible for testing + static final List< + NodeDeprecationCheck> SINGLE_NODE_CHECKS = List.of( + NodeDeprecationChecks::checkMultipleDataPaths, + NodeDeprecationChecks::checkDataPathsList, + NodeDeprecationChecks::checkSharedDataPathSetting, + NodeDeprecationChecks::checkReservedPrefixedRealmNames, + NodeDeprecationChecks::checkExporterUseIngestPipelineSettings, + NodeDeprecationChecks::checkExporterPipelineMasterTimeoutSetting, + NodeDeprecationChecks::checkExporterCreateLegacyTemplateSetting, + NodeDeprecationChecks::checkMonitoringSettingHistoryDuration, + NodeDeprecationChecks::checkMonitoringSettingHistoryDuration, + NodeDeprecationChecks::checkMonitoringSettingCollectIndexRecovery, + NodeDeprecationChecks::checkMonitoringSettingCollectIndices, + NodeDeprecationChecks::checkMonitoringSettingCollectCcrTimeout, + NodeDeprecationChecks::checkMonitoringSettingCollectEnrichStatsTimeout, + NodeDeprecationChecks::checkMonitoringSettingCollectIndexRecoveryStatsTimeout, + NodeDeprecationChecks::checkMonitoringSettingCollectIndexStatsTimeout, + NodeDeprecationChecks::checkMonitoringSettingCollectMlJobStatsTimeout, + NodeDeprecationChecks::checkMonitoringSettingCollectNodeStatsTimeout, + NodeDeprecationChecks::checkMonitoringSettingCollectClusterStatsTimeout, + NodeDeprecationChecks::checkMonitoringSettingExportersHost, + NodeDeprecationChecks::checkMonitoringSettingExportersBulkTimeout, + NodeDeprecationChecks::checkMonitoringSettingExportersConnectionTimeout, + NodeDeprecationChecks::checkMonitoringSettingExportersConnectionReadTimeout, + NodeDeprecationChecks::checkMonitoringSettingExportersAuthUsername, + NodeDeprecationChecks::checkMonitoringSettingExportersAuthPass, + NodeDeprecationChecks::checkMonitoringSettingExportersSSL, + NodeDeprecationChecks::checkMonitoringSettingExportersProxyBase, + NodeDeprecationChecks::checkMonitoringSettingExportersSniffEnabled, + NodeDeprecationChecks::checkMonitoringSettingExportersHeaders, + NodeDeprecationChecks::checkMonitoringSettingExportersTemplateTimeout, + NodeDeprecationChecks::checkMonitoringSettingExportersMasterTimeout, + NodeDeprecationChecks::checkMonitoringSettingExportersEnabled, + NodeDeprecationChecks::checkMonitoringSettingExportersType, + NodeDeprecationChecks::checkMonitoringSettingExportersAlertsEnabled, + NodeDeprecationChecks::checkMonitoringSettingExportersAlertsBlacklist, + NodeDeprecationChecks::checkMonitoringSettingExportersIndexNameTimeFormat, + NodeDeprecationChecks::checkMonitoringSettingDecommissionAlerts, + NodeDeprecationChecks::checkMonitoringSettingEsCollectionEnabled, + NodeDeprecationChecks::checkMonitoringSettingCollectionEnabled, + NodeDeprecationChecks::checkMonitoringSettingCollectionInterval, + NodeDeprecationChecks::checkScriptContextCache, + NodeDeprecationChecks::checkScriptContextCompilationsRateLimitSetting, + NodeDeprecationChecks::checkScriptContextCacheSizeSetting, + NodeDeprecationChecks::checkScriptContextCacheExpirationSetting, + NodeDeprecationChecks::checkEnforceDefaultTierPreferenceSetting, + NodeDeprecationChecks::checkLifecyleStepMasterTimeoutSetting, + NodeDeprecationChecks::checkEqlEnabledSetting, + NodeDeprecationChecks::checkNodeAttrData, + NodeDeprecationChecks::checkWatcherBulkConcurrentRequestsSetting, + NodeDeprecationChecks::checkTracingApmSettings + ); + static DeprecationIssue checkDeprecatedSetting( final Settings clusterSettings, final Settings nodeSettings, @@ -77,15 +131,6 @@ private static Map createMetaMapForRemovableSettings(boolean can return canAutoRemoveSetting ? DeprecationIssue.createMetaMapForRemovableSettings(removableSettings) : null; } - static DeprecationIssue checkRemovedSetting( - final Settings clusterSettings, - final Settings nodeSettings, - final Setting removedSetting, - final String url - ) { - return checkRemovedSetting(clusterSettings, nodeSettings, removedSetting, url, null, DeprecationIssue.Level.CRITICAL); - } - static DeprecationIssue checkRemovedSetting( final Settings clusterSettings, final Settings nodeSettings, @@ -1012,4 +1057,9 @@ static DeprecationIssue checkTracingApmSettings( DeprecationIssue.Level.CRITICAL ); } + + @FunctionalInterface + public interface NodeDeprecationCheck { + R apply(A first, B second, C third, D fourth); + } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ResourceDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ResourceDeprecationChecker.java index 71b9903f69f86..daa3514e3b989 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ResourceDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ResourceDeprecationChecker.java @@ -24,8 +24,14 @@ public interface ResourceDeprecationChecker { * This runs the checks for the current deprecation checker. * * @param clusterState The cluster state provided for the checker + * @param request The deprecation request that triggered this check + * @param precomputedData Data that have been remotely retrieved and might be useful in the checks */ - Map> check(ClusterState clusterState, DeprecationInfoAction.Request request); + Map> check( + ClusterState clusterState, + DeprecationInfoAction.Request request, + TransportDeprecationInfoAction.PrecomputedData precomputedData + ); /** * @return The name of the checker diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationChecker.java index 5a451a9613797..d68e31a1fde4e 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationChecker.java @@ -19,9 +19,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.function.Function; -import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.filterChecks; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_COMMON_DETAIL; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_HELP_URL; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_MESSAGE; @@ -32,20 +32,34 @@ public class TemplateDeprecationChecker implements ResourceDeprecationChecker { public static final String NAME = "templates"; - private static final List> INDEX_TEMPLATE_CHECKS = List.of( - TemplateDeprecationChecker::checkLegacyTiersInIndexTemplate + private final List> indexTemplateChecks = List.of( + this::checkLegacyTiersInIndexTemplate ); - private static final List> COMPONENT_TEMPLATE_CHECKS = List.of( - TemplateDeprecationChecker::checkSourceModeInComponentTemplates, - TemplateDeprecationChecker::checkLegacyTiersInComponentTemplates + private final List> componentTemplateChecks = List.of( + this::checkSourceModeInComponentTemplates, + this::checkLegacyTiersInComponentTemplates ); /** * @param clusterState The cluster state provided for the checker + * @param request not used yet in these checks + * @param precomputedData not used yet in these checks * @return the name of the data streams that have violated the checks with their respective warnings. */ @Override - public Map> check(ClusterState clusterState, DeprecationInfoAction.Request request) { + public Map> check( + ClusterState clusterState, + DeprecationInfoAction.Request request, + TransportDeprecationInfoAction.PrecomputedData precomputedData + ) { + return check(clusterState); + } + + /** + * @param clusterState The cluster state provided for the checker + * @return the name of the data streams that have violated the checks with their respective warnings. + */ + Map> check(ClusterState clusterState) { var indexTemplates = clusterState.metadata().templatesV2().entrySet(); var componentTemplates = clusterState.metadata().componentTemplates().entrySet(); if (indexTemplates.isEmpty() && componentTemplates.isEmpty()) { @@ -56,7 +70,10 @@ public Map> check(ClusterState clusterState, Depr String name = entry.getKey(); ComposableIndexTemplate template = entry.getValue(); - List issuesForSingleIndexTemplate = filterChecks(INDEX_TEMPLATE_CHECKS, c -> c.apply(template)); + List issuesForSingleIndexTemplate = indexTemplateChecks.stream() + .map(c -> c.apply(template)) + .filter(Objects::nonNull) + .toList(); if (issuesForSingleIndexTemplate.isEmpty() == false) { issues.computeIfAbsent(name, ignored -> new ArrayList<>()).addAll(issuesForSingleIndexTemplate); } @@ -65,7 +82,10 @@ public Map> check(ClusterState clusterState, Depr String name = entry.getKey(); ComponentTemplate template = entry.getValue(); - List issuesForSingleIndexTemplate = filterChecks(COMPONENT_TEMPLATE_CHECKS, c -> c.apply(template)); + List issuesForSingleIndexTemplate = componentTemplateChecks.stream() + .map(c -> c.apply(template)) + .filter(Objects::nonNull) + .toList(); if (issuesForSingleIndexTemplate.isEmpty() == false) { issues.computeIfAbsent(name, ignored -> new ArrayList<>()).addAll(issuesForSingleIndexTemplate); } @@ -73,7 +93,7 @@ public Map> check(ClusterState clusterState, Depr return issues.isEmpty() ? Map.of() : issues; } - static DeprecationIssue checkLegacyTiersInIndexTemplate(ComposableIndexTemplate composableIndexTemplate) { + private DeprecationIssue checkLegacyTiersInIndexTemplate(ComposableIndexTemplate composableIndexTemplate) { Template template = composableIndexTemplate.template(); if (template != null) { List deprecatedSettings = LegacyTiersDetection.getDeprecatedFilteredAllocationSettings(template.settings()); @@ -93,7 +113,7 @@ static DeprecationIssue checkLegacyTiersInIndexTemplate(ComposableIndexTemplate return null; } - static DeprecationIssue checkSourceModeInComponentTemplates(ComponentTemplate template) { + private DeprecationIssue checkSourceModeInComponentTemplates(ComponentTemplate template) { if (template.template().mappings() != null) { var sourceAsMap = (Map) XContentHelper.convertToMap(template.template().mappings().uncompressed(), true).v2().get("_doc"); if (sourceAsMap != null) { @@ -115,7 +135,7 @@ static DeprecationIssue checkSourceModeInComponentTemplates(ComponentTemplate te return null; } - static DeprecationIssue checkLegacyTiersInComponentTemplates(ComponentTemplate componentTemplate) { + private DeprecationIssue checkLegacyTiersInComponentTemplates(ComponentTemplate componentTemplate) { Template template = componentTemplate.template(); List deprecatedSettings = LegacyTiersDetection.getDeprecatedFilteredAllocationSettings(template.settings()); if (deprecatedSettings.isEmpty()) { diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransformDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransformDeprecationChecker.java deleted file mode 100644 index 57c4fae960854..0000000000000 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransformDeprecationChecker.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.deprecation; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; -import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; - -import java.util.ArrayList; -import java.util.List; - -class TransformDeprecationChecker implements DeprecationChecker { - - public static final String TRANSFORM_DEPRECATION_KEY = "transform_settings"; - private final List transformConfigs; - - TransformDeprecationChecker(List transformConfigs) { - this.transformConfigs = transformConfigs; - } - - @Override - public boolean enabled(Settings settings) { - // always enabled - return true; - } - - @Override - public void check(Components components, ActionListener deprecationIssueListener) { - ActionListener.completeWith(deprecationIssueListener, () -> { - List allIssues = new ArrayList<>(); - for (var config : transformConfigs) { - allIssues.addAll(config.checkForDeprecations(components.xContentRegistry())); - } - return new CheckResult(getName(), allIssues); - }); - } - - @Override - public String getName() { - return TRANSFORM_DEPRECATION_KEY; - } -} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java index 886eddf82149e..c30d8829c23f3 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java @@ -6,12 +6,11 @@ */ package org.elasticsearch.xpack.deprecation; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.GroupedActionListener; +import org.elasticsearch.action.support.RefCountingListener; import org.elasticsearch.action.support.ThreadedActionListener; import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; import org.elasticsearch.client.internal.OriginSettingClient; @@ -19,14 +18,22 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Tuple; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.transport.Transports; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.action.util.PageParams; @@ -35,24 +42,30 @@ import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.elasticsearch.xpack.deprecation.DeprecationChecks.CLUSTER_SETTINGS_CHECKS; - public class TransportDeprecationInfoAction extends TransportMasterNodeReadAction< DeprecationInfoAction.Request, DeprecationInfoAction.Response> { - private static final DeprecationChecker ML_CHECKER = new MlDeprecationChecker(); - private static final Logger logger = LogManager.getLogger(TransportDeprecationInfoAction.class); + public static final Setting> SKIP_DEPRECATIONS_SETTING = Setting.stringListSetting( + "deprecation.skip_deprecated_settings", + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + private static final List PLUGIN_CHECKERS = List.of(new MlDeprecationChecker()); private final NodeClient client; private final IndexNameExpressionResolver indexNameExpressionResolver; private final Settings settings; private final NamedXContentRegistry xContentRegistry; private volatile List skipTheseDeprecations; + private final NodeDeprecationChecker nodeDeprecationChecker; + private final ClusterDeprecationChecker clusterDeprecationChecker; + private final List resourceDeprecationCheckers; @Inject public TransportDeprecationInfoAction( @@ -79,10 +92,17 @@ public TransportDeprecationInfoAction( this.indexNameExpressionResolver = indexNameExpressionResolver; this.settings = settings; this.xContentRegistry = xContentRegistry; - skipTheseDeprecations = DeprecationChecks.SKIP_DEPRECATIONS_SETTING.get(settings); + skipTheseDeprecations = SKIP_DEPRECATIONS_SETTING.get(settings); + nodeDeprecationChecker = new NodeDeprecationChecker(threadPool); + clusterDeprecationChecker = new ClusterDeprecationChecker(xContentRegistry); + resourceDeprecationCheckers = List.of( + new IndexDeprecationChecker(indexNameExpressionResolver), + new DataStreamDeprecationChecker(indexNameExpressionResolver), + new TemplateDeprecationChecker(), + new IlmPolicyDeprecationChecker() + ); // Safe to register this here because it happens synchronously before the cluster service is started: - clusterService.getClusterSettings() - .addSettingsUpdateConsumer(DeprecationChecks.SKIP_DEPRECATIONS_SETTING, this::setSkipDeprecations); + clusterService.getClusterSettings().addSettingsUpdateConsumer(SKIP_DEPRECATIONS_SETTING, this::setSkipDeprecations); } private void setSkipDeprecations(List skipDeprecations) { @@ -102,56 +122,222 @@ protected final void masterOperation( ClusterState state, final ActionListener listener ) { - NodesDeprecationCheckRequest nodeDepReq = new NodesDeprecationCheckRequest("_all"); - ClientHelper.executeAsyncWithOrigin( - client, - ClientHelper.DEPRECATION_ORIGIN, - NodesDeprecationCheckAction.INSTANCE, - nodeDepReq, - listener.delegateFailureAndWrap((l, response) -> { - if (response.hasFailures()) { - List failedNodeIds = response.failures() - .stream() - .map(failure -> failure.nodeId() + ": " + failure.getMessage()) - .collect(Collectors.toList()); - logger.warn("nodes failed to run deprecation checks: {}", failedNodeIds); - for (FailedNodeException failure : response.failures()) { - logger.debug("node {} failed to run deprecation checks: {}", failure.nodeId(), failure); - } - } - transformConfigs(l.delegateFailureAndWrap((ll, transformConfigs) -> { - DeprecationChecker.Components components = new DeprecationChecker.Components( - xContentRegistry, - settings, - new OriginSettingClient(client, ClientHelper.DEPRECATION_ORIGIN) - ); - pluginSettingIssues( - List.of(ML_CHECKER, new TransformDeprecationChecker(transformConfigs)), - components, - new ThreadedActionListener<>( - client.threadPool().generic(), - ll.map( - deprecationIssues -> DeprecationInfoAction.Response.from( - state, - indexNameExpressionResolver, - request, - response, - CLUSTER_SETTINGS_CHECKS, - deprecationIssues, - skipTheseDeprecations, - List.of( - new IndexDeprecationChecker(indexNameExpressionResolver, indexToTransformIds(transformConfigs)), - new DataStreamDeprecationChecker(indexNameExpressionResolver), - new TemplateDeprecationChecker(), - new IlmPolicyDeprecationChecker() - ) - ) - ) - ) - ); - })); - }) + PrecomputedData precomputedData = new PrecomputedData(); + try (var refs = new RefCountingListener(checkAndCreateResponse(state, request, precomputedData, listener))) { + nodeDeprecationChecker.check(client, refs.acquire(precomputedData::setOnceNodeSettingsIssues)); + transformConfigs(refs.acquire(precomputedData::setOnceTransformConfigs)); + DeprecationChecker.Components components = new DeprecationChecker.Components( + xContentRegistry, + settings, + new OriginSettingClient(client, ClientHelper.DEPRECATION_ORIGIN) + ); + pluginSettingIssues(PLUGIN_CHECKERS, components, refs.acquire(precomputedData::setOncePluginIssues)); + } + } + + /** + * This is the function that does the bulk of the logic of combining the necessary dependencies together, including the cluster state, + * the precalculated information in {@code context} with the remaining checkers such as the cluster setting checker and the resource + * checkers.This function will run a significant part of the checks and build out the final list of issues that exist in the + * cluster. Because of that, it's important that it does not run in the transport thread that's why it's combined with + * {@link #executeInGenericThreadpool(ActionListener)}. + * + * @param state The cluster state + * @param request The originating request containing the index expressions to evaluate + * @param precomputedData Data from remote requests necessary to construct the response + * @param responseListener The listener expecting the {@link DeprecationInfoAction.Response} + * @return The listener that should be executed after all the remote requests have completed and the {@link PrecomputedData} + * is initialised. + */ + public ActionListener checkAndCreateResponse( + ClusterState state, + DeprecationInfoAction.Request request, + PrecomputedData precomputedData, + ActionListener responseListener + ) { + return executeInGenericThreadpool( + ActionListener.running( + () -> responseListener.onResponse( + checkAndCreateResponse( + state, + indexNameExpressionResolver, + request, + skipTheseDeprecations, + clusterDeprecationChecker, + resourceDeprecationCheckers, + precomputedData + ) + ) + ) + ); + } + + /** + * This is the function that does the bulk of the logic of combining the necessary dependencies together, including the cluster state, + * the precalculated information in {@code context} with the remaining checkers such as the cluster setting checker and the resource + * checkers.This function will run a significant part of the checks and build out the final list of issues that exist in the + * cluster. It's important that it does not run in the transport thread that's why it's combined with + * {@link #checkAndCreateResponse(ClusterState, DeprecationInfoAction.Request, PrecomputedData, ActionListener)}. We keep this separated + * for testing purposes. + * + * @param state The cluster state + * @param indexNameExpressionResolver Used to resolve indices into their concrete names + * @param request The originating request containing the index expressions to evaluate + * @param skipTheseDeprecatedSettings the settings that will be removed from cluster metadata and the index metadata of all the + * indexes specified by indexNames + * @param clusterDeprecationChecker The checker that provides the cluster settings deprecations warnings + * @param resourceDeprecationCheckers these are checkers that take as input the cluster state and return a map from resource type + * to issues grouped by the resource name. + * @param precomputedData data from remote requests necessary to construct the response + * @return The list of deprecation issues found in the cluster + */ + static DeprecationInfoAction.Response checkAndCreateResponse( + ClusterState state, + IndexNameExpressionResolver indexNameExpressionResolver, + DeprecationInfoAction.Request request, + List skipTheseDeprecatedSettings, + ClusterDeprecationChecker clusterDeprecationChecker, + List resourceDeprecationCheckers, + PrecomputedData precomputedData + ) { + assert Transports.assertNotTransportThread("walking mappings in indexSettingsChecks is expensive"); + // Allow system index access here to prevent deprecation warnings when we call this API + String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames(state, request); + ClusterState stateWithSkippedSettingsRemoved = removeSkippedSettings(state, concreteIndexNames, skipTheseDeprecatedSettings); + List clusterSettingsIssues = clusterDeprecationChecker.check( + stateWithSkippedSettingsRemoved, + precomputedData.transformConfigs() + ); + + Map>> resourceDeprecationIssues = new HashMap<>(); + for (ResourceDeprecationChecker resourceDeprecationChecker : resourceDeprecationCheckers) { + Map> issues = resourceDeprecationChecker.check( + stateWithSkippedSettingsRemoved, + request, + precomputedData + ); + if (issues.isEmpty() == false) { + resourceDeprecationIssues.put(resourceDeprecationChecker.getName(), issues); + } + } + + return new DeprecationInfoAction.Response( + clusterSettingsIssues, + precomputedData.nodeSettingsIssues(), + resourceDeprecationIssues, + precomputedData.pluginIssues() + ); + } + + /** + * This class holds the results of remote requests. These can be either checks that require remote requests such as + * {@code nodeSettingsIssues} and {@code pluginIssues} or metadata needed for more than one types of checks such as + * {@code transformConfigs}. + */ + public static class PrecomputedData { + private final SetOnce> nodeSettingsIssues = new SetOnce<>(); + private final SetOnce>> pluginIssues = new SetOnce<>(); + private final SetOnce> transformConfigs = new SetOnce<>(); + + public void setOnceNodeSettingsIssues(List nodeSettingsIssues) { + this.nodeSettingsIssues.set(nodeSettingsIssues); + } + + public void setOncePluginIssues(Map> pluginIssues) { + this.pluginIssues.set(pluginIssues); + } + + public void setOnceTransformConfigs(List transformConfigs) { + this.transformConfigs.set(transformConfigs); + } + + public List nodeSettingsIssues() { + return nodeSettingsIssues.get(); + } + + public Map> pluginIssues() { + return pluginIssues.get(); + } + + public List transformConfigs() { + return transformConfigs.get(); + } + } + + /** + * Removes the skipped settings from the selected indices and the component and index templates. + * @param state The cluster state to modify + * @param indexNames The names of the indexes whose settings need to be filtered + * @param skipTheseDeprecatedSettings The settings that will be removed from cluster metadata and the index metadata of all the + * indexes specified by indexNames + * @return A modified cluster state with the given settings removed + */ + private static ClusterState removeSkippedSettings(ClusterState state, String[] indexNames, List skipTheseDeprecatedSettings) { + // Short-circuit, no need to reconstruct the cluster state if there are no settings to remove + if (skipTheseDeprecatedSettings == null || skipTheseDeprecatedSettings.isEmpty()) { + return state; + } + ClusterState.Builder clusterStateBuilder = new ClusterState.Builder(state); + Metadata.Builder metadataBuilder = Metadata.builder(state.metadata()); + metadataBuilder.transientSettings( + metadataBuilder.transientSettings().filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false) + ); + metadataBuilder.persistentSettings( + metadataBuilder.persistentSettings().filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false) ); + Map indicesBuilder = new HashMap<>(state.getMetadata().indices()); + for (String indexName : indexNames) { + IndexMetadata indexMetadata = state.getMetadata().index(indexName); + IndexMetadata.Builder filteredIndexMetadataBuilder = new IndexMetadata.Builder(indexMetadata); + Settings filteredSettings = indexMetadata.getSettings() + .filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false); + filteredIndexMetadataBuilder.settings(filteredSettings); + indicesBuilder.put(indexName, filteredIndexMetadataBuilder.build()); + } + metadataBuilder.componentTemplates(state.metadata().componentTemplates().entrySet().stream().map(entry -> { + String templateName = entry.getKey(); + ComponentTemplate componentTemplate = entry.getValue(); + Template template = componentTemplate.template(); + if (template.settings() == null || template.settings().isEmpty()) { + return Tuple.tuple(templateName, componentTemplate); + } + return Tuple.tuple( + templateName, + new ComponentTemplate( + Template.builder(template) + .settings(template.settings().filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false)) + .build(), + componentTemplate.version(), + componentTemplate.metadata(), + componentTemplate.deprecated() + ) + ); + }).collect(Collectors.toMap(Tuple::v1, Tuple::v2))); + metadataBuilder.indexTemplates(state.metadata().templatesV2().entrySet().stream().map(entry -> { + String templateName = entry.getKey(); + ComposableIndexTemplate indexTemplate = entry.getValue(); + Template template = indexTemplate.template(); + if (template == null || template.settings() == null || template.settings().isEmpty()) { + return Tuple.tuple(templateName, indexTemplate); + } + return Tuple.tuple( + templateName, + indexTemplate.toBuilder() + .template( + Template.builder(indexTemplate.template()) + .settings( + indexTemplate.template() + .settings() + .filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false) + ) + ) + .build() + ); + }).collect(Collectors.toMap(Tuple::v1, Tuple::v2))); + + metadataBuilder.indices(indicesBuilder); + clusterStateBuilder.metadata(metadataBuilder); + return clusterStateBuilder.build(); } static void pluginSettingIssues( @@ -192,34 +378,21 @@ private void transformConfigs(PageParams currentPage, ActionListener( - threadPool.generic(), - currentPageListener.delegateFailureAndWrap((delegate, getTransformConfigResponse) -> { - var currentPageOfConfigs = getTransformConfigResponse.getTransformConfigurations().stream(); - var currentPageSize = currentPage.getFrom() + currentPage.getSize(); - var totalTransformConfigCount = getTransformConfigResponse.getTransformConfigurationCount(); - if (totalTransformConfigCount >= currentPageSize) { - var nextPage = new PageParams(currentPageSize, PageParams.DEFAULT_SIZE); - transformConfigs( - nextPage, - delegate.map(nextPageOfConfigs -> Stream.concat(currentPageOfConfigs, nextPageOfConfigs)) - ); - } else { - delegate.onResponse(currentPageOfConfigs); - } - }) - ) + executeInGenericThreadpool(currentPageListener.delegateFailureAndWrap((delegate, getTransformConfigResponse) -> { + var currentPageOfConfigs = getTransformConfigResponse.getTransformConfigurations().stream(); + var currentPageSize = currentPage.getFrom() + currentPage.getSize(); + var totalTransformConfigCount = getTransformConfigResponse.getTransformConfigurationCount(); + if (totalTransformConfigCount >= currentPageSize) { + var nextPage = new PageParams(currentPageSize, PageParams.DEFAULT_SIZE); + transformConfigs(nextPage, delegate.map(nextPageOfConfigs -> Stream.concat(currentPageOfConfigs, nextPageOfConfigs))); + } else { + delegate.onResponse(currentPageOfConfigs); + } + })) ); } - private Map> indexToTransformIds(List transformConfigs) { - return transformConfigs.stream() - .collect( - Collectors.groupingBy( - config -> config.getDestination().getIndex(), - Collectors.mapping(TransformConfig::getId, Collectors.toList()) - ) - ); + private ActionListener executeInGenericThreadpool(ActionListener listener) { + return new ThreadedActionListener<>(threadPool.generic(), listener); } - } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java index 745f5e7ae8959..befe0bd6b41a4 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java @@ -36,6 +36,7 @@ import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.Objects; import static org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING; @@ -75,10 +76,10 @@ public TransportNodeDeprecationCheckAction( this.pluginsService = pluginsService; this.licenseState = licenseState; this.clusterInfoService = clusterInfoService; - skipTheseDeprecations = DeprecationChecks.SKIP_DEPRECATIONS_SETTING.get(settings); + skipTheseDeprecations = TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING.get(settings); // Safe to register this here because it happens synchronously before the cluster service is started: clusterService.getClusterSettings() - .addSettingsUpdateConsumer(DeprecationChecks.SKIP_DEPRECATIONS_SETTING, this::setSkipDeprecations); + .addSettingsUpdateConsumer(TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING, this::setSkipDeprecations); } private void setSkipDeprecations(List skipDeprecations) { @@ -106,13 +107,13 @@ protected NodesDeprecationCheckAction.NodeResponse newNodeResponse(StreamInput i @Override protected NodesDeprecationCheckAction.NodeResponse nodeOperation(NodesDeprecationCheckAction.NodeRequest request, Task task) { - return nodeOperation(request, DeprecationChecks.NODE_SETTINGS_CHECKS); + return nodeOperation(request, NodeDeprecationChecks.SINGLE_NODE_CHECKS); } NodesDeprecationCheckAction.NodeResponse nodeOperation( NodesDeprecationCheckAction.NodeRequest request, List< - DeprecationChecks.NodeDeprecationCheck< + NodeDeprecationChecks.NodeDeprecationCheck< Settings, PluginsAndModules, ClusterState, @@ -130,10 +131,10 @@ NodesDeprecationCheckAction.NodeResponse nodeOperation( .metadata(Metadata.builder(metadata).transientSettings(transientSettings).persistentSettings(persistentSettings).build()) .build(); - List issues = DeprecationInfoAction.filterChecks( - nodeSettingsChecks, - (c) -> c.apply(filteredNodeSettings, pluginsService.info(), filteredClusterState, licenseState) - ); + List issues = nodeSettingsChecks.stream() + .map(c -> c.apply(filteredNodeSettings, pluginsService.info(), filteredClusterState, licenseState)) + .filter(Objects::nonNull) + .toList(); DeprecationIssue watermarkIssue = checkDiskLowWatermark( filteredNodeSettings, filteredClusterState.metadata().settings(), diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java index 450dee7a76a98..a8dd1d464e30c 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java @@ -69,7 +69,7 @@ public void testOldIndicesCheck() { ); // We know that the data stream checks ignore the request. - Map> issuesByDataStream = checker.check(clusterState, null); + Map> issuesByDataStream = checker.check(clusterState); assertThat(issuesByDataStream.size(), equalTo(1)); assertThat(issuesByDataStream.containsKey(dataStream.getName()), equalTo(true)); assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); @@ -91,7 +91,7 @@ public void testOldIndicesCheckWithOnlyNewIndices() { .build(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); - Map> issuesByDataStream = checker.check(clusterState, null); + Map> issuesByDataStream = checker.check(clusterState); assertThat(issuesByDataStream.size(), equalTo(0)); } @@ -137,7 +137,7 @@ public void testOldIndicesCheckWithClosedAndOpenIndices() { ) ); - Map> issuesByDataStream = checker.check(clusterState, null); + Map> issuesByDataStream = checker.check(clusterState); assertThat(issuesByDataStream.containsKey(dataStream.getName()), equalTo(true)); assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); } @@ -297,7 +297,7 @@ public void testOldIndicesIgnoredWarningCheck() { ) ); - Map> issuesByDataStream = checker.check(clusterState, null); + Map> issuesByDataStream = checker.check(clusterState); assertThat(issuesByDataStream.containsKey(dataStream.getName()), equalTo(true)); assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationChecksTests.java deleted file mode 100644 index 9a57450b7fad7..0000000000000 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationChecksTests.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.deprecation; - -import org.elasticsearch.core.Tuple; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; - -import java.util.ArrayList; -import java.util.List; -import java.util.function.Supplier; - -import static org.hamcrest.Matchers.equalTo; - -public class DeprecationChecksTests extends ESTestCase { - - public void testFilterChecks() { - DeprecationIssue issue = createRandomDeprecationIssue(); - int numChecksPassed = randomIntBetween(0, 5); - int numChecksFailed = 10 - numChecksPassed; - List> checks = new ArrayList<>(); - for (int i = 0; i < numChecksFailed; i++) { - checks.add(() -> issue); - } - for (int i = 0; i < numChecksPassed; i++) { - checks.add(() -> null); - } - List filteredIssues = DeprecationInfoAction.filterChecks(checks, Supplier::get); - assertThat(filteredIssues.size(), equalTo(numChecksFailed)); - } - - private static DeprecationIssue createRandomDeprecationIssue() { - String details = randomBoolean() ? randomAlphaOfLength(10) : null; - return new DeprecationIssue( - randomFrom(DeprecationIssue.Level.values()), - randomAlphaOfLength(10), - randomAlphaOfLength(10), - details, - randomBoolean(), - randomMap(1, 5, () -> Tuple.tuple(randomAlphaOfLength(4), randomAlphaOfLength(4))) - ); - } -} diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java index a06bb2f2718cb..537c3eb84a902 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java @@ -6,50 +6,17 @@ */ package org.elasticsearch.xpack.deprecation; -import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.ComponentTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.metadata.Template; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNodeUtils; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.core.Tuple; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue.Level; -import org.junit.Assert; -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.BiFunction; -import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.Response.RESERVED_NAMES; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.core.IsEqual.equalTo; - public class DeprecationInfoActionResponseTests extends AbstractWireSerializingTestCase { @Override @@ -153,329 +120,11 @@ protected Writeable.Reader instanceReader() { return DeprecationInfoAction.Response::new; } - public void testFrom() throws IOException { - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all"); - mapping.field("enabled", false); - mapping.endObject().endObject(); - - Metadata metadata = Metadata.builder() - .put( - IndexMetadata.builder("test") - .putMapping(Strings.toString(mapping)) - .settings(settings(IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - ) - .build(); - - DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("test", new TransportAddress(TransportAddress.META_ADDRESS, 9300)); - ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); - IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); - boolean clusterIssueFound = randomBoolean(); - boolean nodeIssueFound = randomBoolean(); - boolean indexIssueFound = randomBoolean(); - boolean dataStreamIssueFound = randomBoolean(); - boolean indexTemplateIssueFound = randomBoolean(); - boolean componentTemplateIssueFound = randomBoolean(); - boolean ilmPolicyIssueFound = randomBoolean(); - DeprecationIssue foundIssue = createTestDeprecationIssue(); - List> clusterSettingsChecks = List.of((s) -> clusterIssueFound ? foundIssue : null); - List resourceCheckers = List.of(createResourceChecker("index_settings", (cs, req) -> { - if (indexIssueFound) { - return Map.of("test", List.of(foundIssue)); - } - return Map.of(); - }), createResourceChecker("data_streams", (cs, req) -> { - if (dataStreamIssueFound) { - return Map.of("my-ds", List.of(foundIssue)); - } - return Map.of(); - }), createResourceChecker("templates", (cs, req) -> { - Map> issues = new HashMap<>(); - if (componentTemplateIssueFound) { - issues.put("my-component-template", List.of(foundIssue)); - } - if (indexTemplateIssueFound) { - issues.put("my-index-template", List.of(foundIssue)); - } - return issues; - }), createResourceChecker("ilm_policies", (cs, req) -> { - if (ilmPolicyIssueFound) { - return Map.of("my-policy", List.of(foundIssue)); - } - return Map.of(); - })); - - NodesDeprecationCheckResponse nodeDeprecationIssues = new NodesDeprecationCheckResponse( - new ClusterName(randomAlphaOfLength(5)), - nodeIssueFound ? List.of(new NodesDeprecationCheckAction.NodeResponse(discoveryNode, List.of(foundIssue))) : List.of(), - List.of() - ); - - DeprecationInfoAction.Request request = new DeprecationInfoAction.Request(randomTimeValue(), Strings.EMPTY_ARRAY); - DeprecationInfoAction.Response response = DeprecationInfoAction.Response.from( - state, - resolver, - request, - nodeDeprecationIssues, - clusterSettingsChecks, - new HashMap<>(), // modified in the method to move transform deprecation issues into cluster_settings - List.of(), - resourceCheckers - ); - - if (clusterIssueFound) { - assertThat(response.getClusterSettingsIssues(), equalTo(List.of(foundIssue))); - } else { - assertThat(response.getClusterSettingsIssues(), empty()); - } - - if (nodeIssueFound) { - String details = foundIssue.getDetails() != null ? foundIssue.getDetails() + " " : ""; - DeprecationIssue mergedFoundIssue = new DeprecationIssue( - foundIssue.getLevel(), - foundIssue.getMessage(), - foundIssue.getUrl(), - details + "(nodes impacted: [" + discoveryNode.getName() + "])", - foundIssue.isResolveDuringRollingUpgrade(), - foundIssue.getMeta() - ); - assertThat(response.getNodeSettingsIssues(), equalTo(List.of(mergedFoundIssue))); - } else { - assertTrue(response.getNodeSettingsIssues().isEmpty()); - } - - if (indexIssueFound) { - assertThat(response.getIndexSettingsIssues(), equalTo(Map.of("test", List.of(foundIssue)))); - } else { - assertTrue(response.getIndexSettingsIssues().isEmpty()); - } - if (dataStreamIssueFound) { - assertThat(response.getDataStreamDeprecationIssues(), equalTo(Map.of("my-ds", List.of(foundIssue)))); - } else { - assertTrue(response.getDataStreamDeprecationIssues().isEmpty()); - } - if (ilmPolicyIssueFound) { - assertThat(response.getIlmPolicyDeprecationIssues(), equalTo(Map.of("my-policy", List.of(foundIssue)))); - } else { - assertTrue(response.getIlmPolicyDeprecationIssues().isEmpty()); - } - if (componentTemplateIssueFound == false && indexTemplateIssueFound == false) { - assertTrue(response.getTemplateDeprecationIssues().isEmpty()); - } else { - if (componentTemplateIssueFound) { - assertThat(response.getTemplateDeprecationIssues().get("my-component-template"), equalTo(List.of(foundIssue))); - } - if (indexTemplateIssueFound) { - assertThat(response.getTemplateDeprecationIssues().get("my-index-template"), equalTo(List.of(foundIssue))); - } - - } - } - - public void testFromWithMergeableNodeIssues() throws IOException { - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all"); - mapping.field("enabled", false); - mapping.endObject().endObject(); - - Metadata metadata = Metadata.builder() - .put( - IndexMetadata.builder("test") - .putMapping(Strings.toString(mapping)) - .settings(settings(IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - ) - .build(); - - DiscoveryNode node1 = DiscoveryNodeUtils.builder("nodeId1") - .name("node1") - .ephemeralId("ephemeralId1") - .address("hostName1", "hostAddress1", new TransportAddress(TransportAddress.META_ADDRESS, 9300)) - .roles(Set.of()) - .build(); - DiscoveryNode node2 = DiscoveryNodeUtils.builder("nodeId2") - .name("node2") - .ephemeralId("ephemeralId2") - .address("hostName2", "hostAddress2", new TransportAddress(TransportAddress.META_ADDRESS, 9500)) - .roles(Set.of()) - .build(); - ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); - IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); - Map metaMap1 = DeprecationIssue.createMetaMapForRemovableSettings(List.of("setting.1", "setting.2", "setting.3")); - Map metaMap2 = DeprecationIssue.createMetaMapForRemovableSettings(List.of("setting.2", "setting.3")); - DeprecationIssue foundIssue1 = createTestDeprecationIssue(metaMap1); - DeprecationIssue foundIssue2 = createTestDeprecationIssue(foundIssue1, metaMap2); - List> clusterSettingsChecks = List.of(); - List resourceCheckers = List.of(); - - NodesDeprecationCheckResponse nodeDeprecationIssues = new NodesDeprecationCheckResponse( - new ClusterName(randomAlphaOfLength(5)), - Arrays.asList( - new NodesDeprecationCheckAction.NodeResponse(node1, List.of(foundIssue1)), - new NodesDeprecationCheckAction.NodeResponse(node2, List.of(foundIssue2)) - ), - List.of() - ); - - DeprecationInfoAction.Request request = new DeprecationInfoAction.Request(randomTimeValue(), Strings.EMPTY_ARRAY); - DeprecationInfoAction.Response response = DeprecationInfoAction.Response.from( - state, - resolver, - request, - nodeDeprecationIssues, - clusterSettingsChecks, - new HashMap<>(), // modified in the method to move transform deprecation issues into cluster_settings - List.of(), - resourceCheckers - ); - - String details = foundIssue1.getDetails() != null ? foundIssue1.getDetails() + " " : ""; - DeprecationIssue mergedFoundIssue = new DeprecationIssue( - foundIssue1.getLevel(), - foundIssue1.getMessage(), - foundIssue1.getUrl(), - details + "(nodes impacted: [" + node1.getName() + ", " + node2.getName() + "])", - foundIssue1.isResolveDuringRollingUpgrade(), - foundIssue2.getMeta() - ); - assertThat(response.getNodeSettingsIssues(), equalTo(List.of(mergedFoundIssue))); - } - - public void testRemoveSkippedSettings() { - Settings.Builder settingsBuilder = settings(IndexVersion.current()); - settingsBuilder.put("some.deprecated.property", "someValue1"); - settingsBuilder.put("some.other.bad.deprecated.property", "someValue2"); - settingsBuilder.put("some.undeprecated.property", "someValue3"); - settingsBuilder.putList("some.undeprecated.list.property", List.of("someValue4", "someValue5")); - Settings inputSettings = settingsBuilder.build(); - IndexMetadata dataStreamIndexMetadata = IndexMetadata.builder("ds-test-index-1") - .settings(inputSettings) - .numberOfShards(1) - .numberOfReplicas(0) - .build(); - ComponentTemplate componentTemplate = new ComponentTemplate(Template.builder().settings(inputSettings).build(), null, null); - ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() - .template(Template.builder().settings(inputSettings)) - .build(); - Metadata metadata = Metadata.builder() - .put(IndexMetadata.builder("test").settings(inputSettings).numberOfShards(1).numberOfReplicas(0)) - .put(dataStreamIndexMetadata, true) - .put(DataStream.builder("ds-test", List.of(dataStreamIndexMetadata.getIndex())).build()) - .indexTemplates( - Map.of( - "my-index-template", - indexTemplate, - "empty-template", - ComposableIndexTemplate.builder().indexPatterns(List.of("random")).build() - ) - ) - .componentTemplates(Map.of("my-component-template", componentTemplate)) - .persistentSettings(inputSettings) - .build(); - - ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); - IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); - AtomicReference visibleClusterSettings = new AtomicReference<>(); - List> clusterSettingsChecks = List.of((s) -> { - visibleClusterSettings.set(s.getMetadata().settings()); - return null; - }); - AtomicReference visibleIndexSettings = new AtomicReference<>(); - AtomicReference visibleComponentTemplateSettings = new AtomicReference<>(); - AtomicReference visibleIndexTemplateSettings = new AtomicReference<>(); - AtomicInteger backingIndicesCount = new AtomicInteger(0); - List resourceCheckers = List.of(createResourceChecker("index_settings", (cs, req) -> { - for (String indexName : resolver.concreteIndexNames(cs, req)) { - visibleIndexSettings.set(cs.metadata().index(indexName).getSettings()); - } - return Map.of(); - }), createResourceChecker("data_streams", (cs, req) -> { - cs.metadata().dataStreams().values().forEach(ds -> backingIndicesCount.set(ds.getIndices().size())); - return Map.of(); - }), createResourceChecker("templates", (cs, req) -> { - cs.metadata() - .componentTemplates() - .values() - .forEach(template -> visibleComponentTemplateSettings.set(template.template().settings())); - cs.metadata().templatesV2().values().forEach(template -> { - if (template.template() != null && template.template().settings() != null) { - visibleIndexTemplateSettings.set(template.template().settings()); - } - }); - return Map.of(); - })); - - NodesDeprecationCheckResponse nodeDeprecationIssues = new NodesDeprecationCheckResponse( - new ClusterName(randomAlphaOfLength(5)), - List.of(), - List.of() - ); - - DeprecationInfoAction.Request request = new DeprecationInfoAction.Request(randomTimeValue(), Strings.EMPTY_ARRAY); - DeprecationInfoAction.Response.from( - state, - resolver, - request, - nodeDeprecationIssues, - clusterSettingsChecks, - new HashMap<>(), // modified in the method to move transform deprecation issues into cluster_settings - List.of("some.deprecated.property", "some.other.*.deprecated.property"), - resourceCheckers - ); - - settingsBuilder = settings(IndexVersion.current()); - settingsBuilder.put("some.undeprecated.property", "someValue3"); - settingsBuilder.putList("some.undeprecated.list.property", List.of("someValue4", "someValue5")); - - Settings expectedSettings = settingsBuilder.build(); - Settings resultClusterSettings = visibleClusterSettings.get(); - Assert.assertNotNull(resultClusterSettings); - Assert.assertEquals(expectedSettings, visibleClusterSettings.get()); - - Settings resultIndexSettings = visibleIndexSettings.get(); - Assert.assertNotNull(resultIndexSettings); - Assert.assertEquals("someValue3", resultIndexSettings.get("some.undeprecated.property")); - Assert.assertEquals(resultIndexSettings.getAsList("some.undeprecated.list.property"), List.of("someValue4", "someValue5")); - Assert.assertFalse(resultIndexSettings.hasValue("some.deprecated.property")); - Assert.assertFalse(resultIndexSettings.hasValue("some.other.bad.deprecated.property")); - - assertThat(backingIndicesCount.get(), equalTo(1)); - - Assert.assertNotNull(visibleComponentTemplateSettings.get()); - Assert.assertEquals(expectedSettings, visibleComponentTemplateSettings.get()); - Assert.assertNotNull(visibleIndexTemplateSettings.get()); - Assert.assertEquals(expectedSettings, visibleIndexTemplateSettings.get()); - } - - public void testCtorFailure() { - Map> indexNames = Stream.generate(() -> randomAlphaOfLength(10)) - .limit(10) - .collect(Collectors.toMap(Function.identity(), (_k) -> List.of())); - Map> dataStreamNames = Stream.generate(() -> randomAlphaOfLength(10)) - .limit(10) - .collect(Collectors.toMap(Function.identity(), (_k) -> List.of())); - Set shouldCauseFailure = new HashSet<>(RESERVED_NAMES); - for (int i = 0; i < NUMBER_OF_TEST_RUNS; i++) { - Map> pluginSettingsIssues = randomSubsetOf(3, shouldCauseFailure).stream() - .collect(Collectors.toMap(Function.identity(), (_k) -> List.of())); - expectThrows( - ElasticsearchStatusException.class, - () -> new DeprecationInfoAction.Response( - List.of(), - List.of(), - Map.of("data_streams", dataStreamNames, "index_settings", indexNames), - pluginSettingsIssues - ) - ); - } - } - - private static DeprecationIssue createTestDeprecationIssue() { + static DeprecationIssue createTestDeprecationIssue() { return createTestDeprecationIssue(randomMap(1, 5, () -> Tuple.tuple(randomAlphaOfLength(4), randomAlphaOfLength(4)))); } - private static DeprecationIssue createTestDeprecationIssue(Map metaMap) { + static DeprecationIssue createTestDeprecationIssue(Map metaMap) { String details = randomBoolean() ? randomAlphaOfLength(10) : null; return new DeprecationIssue( randomFrom(Level.values()), @@ -487,7 +136,7 @@ private static DeprecationIssue createTestDeprecationIssue(Map m ); } - private static DeprecationIssue createTestDeprecationIssue(DeprecationIssue seedIssue, Map metaMap) { + static DeprecationIssue createTestDeprecationIssue(DeprecationIssue seedIssue, Map metaMap) { return new DeprecationIssue( seedIssue.getLevel(), seedIssue.getMessage(), @@ -498,27 +147,9 @@ private static DeprecationIssue createTestDeprecationIssue(DeprecationIssue seed ); } - private static List randomDeprecationIssues() { + static List randomDeprecationIssues() { return Stream.generate(DeprecationInfoActionResponseTests::createTestDeprecationIssue) .limit(randomIntBetween(0, 10)) .collect(Collectors.toList()); } - - private static ResourceDeprecationChecker createResourceChecker( - String name, - BiFunction>> check - ) { - return new ResourceDeprecationChecker() { - - @Override - public Map> check(ClusterState clusterState, DeprecationInfoAction.Request request) { - return check.apply(clusterState, request); - } - - @Override - public String getName() { - return name; - } - }; - } } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationCheckerTests.java index 2032a6faedc92..475cd3e6a24bc 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationCheckerTests.java @@ -90,7 +90,7 @@ public void testLegacyTierSettings() { ) .build(); - Map> issuesByComponentTemplate = checker.check(clusterState, null); + Map> issuesByComponentTemplate = checker.check(clusterState); final DeprecationIssue expected = new DeprecationIssue( DeprecationIssue.Level.WARNING, "Configuring tiers via filtered allocation is not recommended.", @@ -136,7 +136,7 @@ public void testFrozenAction() { ) .build(); - Map> issuesByComponentTemplate = checker.check(clusterState, null); + Map> issuesByComponentTemplate = checker.check(clusterState); final DeprecationIssue expected = new DeprecationIssue( DeprecationIssue.Level.WARNING, "ILM policy [deprecated-action] contains the action 'freeze' that is deprecated and will be removed in a future version.", diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java index ea30c93bfb5b8..296ae31cd9f24 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java @@ -29,7 +29,11 @@ import org.elasticsearch.snapshots.SearchableSnapshotsSettings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; +import org.elasticsearch.xpack.core.transform.transforms.DestConfig; +import org.elasticsearch.xpack.core.transform.transforms.SourceConfig; +import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -42,10 +46,18 @@ public class IndexDeprecationCheckerTests extends ESTestCase { + private static final IndexVersion OLD_VERSION = IndexVersion.fromId(7170099); + private final IndexNameExpressionResolver indexNameExpressionResolver = TestIndexNameExpressionResolver.newInstance(); + private final IndexDeprecationChecker checker = new IndexDeprecationChecker(indexNameExpressionResolver); + private final TransportDeprecationInfoAction.PrecomputedData emptyPrecomputedData = + new TransportDeprecationInfoAction.PrecomputedData(); private final IndexMetadata.State indexMetdataState; public IndexDeprecationCheckerTests(@Name("indexMetadataState") IndexMetadata.State indexMetdataState) { this.indexMetdataState = indexMetdataState; + emptyPrecomputedData.setOnceNodeSettingsIssues(List.of()); + emptyPrecomputedData.setOncePluginIssues(Map.of()); + emptyPrecomputedData.setOnceTransformConfigs(List.of()); } @ParametersFactory @@ -53,11 +65,6 @@ public static List createParameters() { return List.of(new Object[] { IndexMetadata.State.OPEN }, new Object[] { IndexMetadata.State.CLOSE }); } - private static final IndexVersion OLD_VERSION = IndexVersion.fromId(7170099); - - private final IndexNameExpressionResolver indexNameExpressionResolver = TestIndexNameExpressionResolver.newInstance(); - private final IndexDeprecationChecker checker = new IndexDeprecationChecker(indexNameExpressionResolver, Map.of()); - public void testOldIndicesCheck() { IndexMetadata indexMetadata = IndexMetadata.builder("test") .settings(settings(OLD_VERSION)) @@ -79,14 +86,15 @@ public void testOldIndicesCheck() { ); Map> issuesByIndex = checker.check( clusterState, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); List issues = issuesByIndex.get("test"); assertEquals(singletonList(expected), issues); } public void testOldTransformIndicesCheck() { - var checker = new IndexDeprecationChecker(indexNameExpressionResolver, Map.of("test", List.of("test-transform"))); + var checker = new IndexDeprecationChecker(indexNameExpressionResolver); var indexMetadata = indexMetadata("test", OLD_VERSION); var clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) .metadata(Metadata.builder().put(indexMetadata, true)) @@ -100,15 +108,15 @@ public void testOldTransformIndicesCheck() { false, Map.of("reindex_required", true, "transform_ids", List.of("test-transform")) ); - var issuesByIndex = checker.check(clusterState, new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS)); + var issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + createContextWithTransformConfigs(Map.of("test", List.of("test-transform"))) + ); assertEquals(singletonList(expected), issuesByIndex.get("test")); } public void testOldIndicesCheckWithMultipleTransforms() { - var checker = new IndexDeprecationChecker( - indexNameExpressionResolver, - Map.of("test", List.of("test-transform1", "test-transform2")) - ); var indexMetadata = indexMetadata("test", OLD_VERSION); var clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) .metadata(Metadata.builder().put(indexMetadata, true)) @@ -122,15 +130,15 @@ public void testOldIndicesCheckWithMultipleTransforms() { false, Map.of("reindex_required", true, "transform_ids", List.of("test-transform1", "test-transform2")) ); - var issuesByIndex = checker.check(clusterState, new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS)); + var issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + createContextWithTransformConfigs(Map.of("test", List.of("test-transform1", "test-transform2"))) + ); assertEquals(singletonList(expected), issuesByIndex.get("test")); } public void testMultipleOldIndicesCheckWithTransforms() { - var checker = new IndexDeprecationChecker( - indexNameExpressionResolver, - Map.of("test1", List.of("test-transform1"), "test2", List.of("test-transform2")) - ); var indexMetadata1 = indexMetadata("test1", OLD_VERSION); var indexMetadata2 = indexMetadata("test2", OLD_VERSION); var clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) @@ -161,7 +169,11 @@ public void testMultipleOldIndicesCheckWithTransforms() { ) ) ); - var issuesByIndex = checker.check(clusterState, new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS)); + var issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + createContextWithTransformConfigs(Map.of("test1", List.of("test-transform1"), "test2", List.of("test-transform2"))) + ); assertEquals(expected, issuesByIndex); } @@ -215,7 +227,8 @@ public void testOldIndicesCheckDataStreamIndex() { .build(); Map> issuesByIndex = checker.check( clusterState, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); assertThat(issuesByIndex.size(), equalTo(0)); } @@ -236,7 +249,8 @@ public void testOldIndicesCheckSnapshotIgnored() { Map> issuesByIndex = checker.check( clusterState, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); assertThat(issuesByIndex.size(), equalTo(0)); } @@ -263,7 +277,8 @@ public void testOldIndicesIgnoredWarningCheck() { ); Map> issuesByIndex = checker.check( clusterState, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); assertTrue(issuesByIndex.containsKey("test")); assertEquals(List.of(expected), issuesByIndex.get("test")); @@ -285,7 +300,8 @@ public void testTranslogRetentionSettings() { .build(); Map> issuesByIndex = checker.check( state, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); List issues = issuesByIndex.get("test"); assertThat( @@ -328,7 +344,8 @@ public void testDefaultTranslogRetentionSettings() { .build(); Map> issuesByIndex = checker.check( state, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); assertThat(issuesByIndex.size(), equalTo(0)); } @@ -348,7 +365,8 @@ public void testIndexDataPathSetting() { .build(); Map> issuesByIndex = checker.check( state, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); final String expectedUrl = "https://www.elastic.co/guide/en/elasticsearch/reference/7.13/breaking-changes-7.13.html#deprecate-shared-data-path-setting"; @@ -382,7 +400,8 @@ public void testSimpleFSSetting() { .build(); Map> issuesByIndex = checker.check( state, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); assertThat( issuesByIndex.get("test"), @@ -425,7 +444,8 @@ public void testCamelCaseDeprecation() { .build(); Map> issuesByIndex = checker.check( state, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); DeprecationIssue expected = new DeprecationIssue( DeprecationIssue.Level.CRITICAL, @@ -456,7 +476,8 @@ public void testLegacyTierIndex() { .build(); Map> issuesByIndex = checker.check( state, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); assertThat( issuesByIndex.get("test"), @@ -484,4 +505,23 @@ private ClusterBlocks clusterBlocksForIndices(IndexMetadata... indicesMetadatas) } return builder.build(); } + + private TransportDeprecationInfoAction.PrecomputedData createContextWithTransformConfigs(Map> indexToTransform) { + List transforms = new ArrayList<>(); + for (Map.Entry> entry : indexToTransform.entrySet()) { + String index = entry.getKey(); + for (String transform : entry.getValue()) { + transforms.add( + TransformConfig.builder() + .setId(transform) + .setSource(new SourceConfig(randomAlphaOfLength(10))) + .setDest(new DestConfig(index, List.of(), null)) + .build() + ); + } + } + TransportDeprecationInfoAction.PrecomputedData precomputedData = new TransportDeprecationInfoAction.PrecomputedData(); + precomputedData.setOnceTransformConfigs(transforms); + return precomputedData; + } } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationCheckerTests.java new file mode 100644 index 0000000000000..78ddba87b9f85 --- /dev/null +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationCheckerTests.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.deprecation; + +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.xpack.deprecation.DeprecationInfoActionResponseTests.createTestDeprecationIssue; +import static org.hamcrest.core.IsEqual.equalTo; + +public class NodeDeprecationCheckerTests extends ESTestCase { + + public void testMergingNodeIssues() throws IOException { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all"); + mapping.field("enabled", false); + mapping.endObject().endObject(); + + DiscoveryNode node1 = DiscoveryNodeUtils.builder("nodeId1") + .name("node1") + .ephemeralId("ephemeralId1") + .address("hostName1", "hostAddress1", new TransportAddress(TransportAddress.META_ADDRESS, 9300)) + .roles(Set.of()) + .build(); + DiscoveryNode node2 = DiscoveryNodeUtils.builder("nodeId2") + .name("node2") + .ephemeralId("ephemeralId2") + .address("hostName2", "hostAddress2", new TransportAddress(TransportAddress.META_ADDRESS, 9500)) + .roles(Set.of()) + .build(); + Map metaMap1 = DeprecationIssue.createMetaMapForRemovableSettings(List.of("setting.1", "setting.2", "setting.3")); + Map metaMap2 = DeprecationIssue.createMetaMapForRemovableSettings(List.of("setting.2", "setting.3")); + DeprecationIssue foundIssue1 = createTestDeprecationIssue(metaMap1); + DeprecationIssue foundIssue2 = createTestDeprecationIssue(foundIssue1, metaMap2); + + NodesDeprecationCheckResponse nodeDeprecationIssues = new NodesDeprecationCheckResponse( + new ClusterName(randomAlphaOfLength(5)), + Arrays.asList( + new NodesDeprecationCheckAction.NodeResponse(node1, List.of(foundIssue1)), + new NodesDeprecationCheckAction.NodeResponse(node2, List.of(foundIssue2)) + ), + List.of() + ); + + List result = NodeDeprecationChecker.reduceToDeprecationIssues(nodeDeprecationIssues); + + String details = foundIssue1.getDetails() != null ? foundIssue1.getDetails() + " " : ""; + DeprecationIssue mergedFoundIssue = new DeprecationIssue( + foundIssue1.getLevel(), + foundIssue1.getMessage(), + foundIssue1.getUrl(), + details + "(nodes impacted: [" + node1.getName() + ", " + node2.getName() + "])", + foundIssue1.isResolveDuringRollingUpgrade(), + foundIssue2.getMeta() + ); + assertThat(result, equalTo(List.of(mergedFoundIssue))); + } +} diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java index 3aaee0e5cdb52..18b85ff532234 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java @@ -30,9 +30,11 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.function.Function; import java.util.stream.Collectors; -import static org.elasticsearch.xpack.deprecation.DeprecationChecks.NODE_SETTINGS_CHECKS; +import static org.elasticsearch.xpack.deprecation.NodeDeprecationChecks.SINGLE_NODE_CHECKS; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.not; @@ -154,8 +156,8 @@ public void testSharedDataPathSetting() { .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), createTempDir()) .build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); final String expectedUrl = @@ -209,8 +211,8 @@ public void testCheckReservedPrefixedRealmNames() { } final Settings settings = builder.build(); - final List deprecationIssues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + final List deprecationIssues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -235,8 +237,8 @@ public void testCheckReservedPrefixedRealmNames() { void monitoringSetting(String settingKey, String value) { Settings settings = Settings.builder().put(settingKey, value).build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); final String expectedUrl = "https://ela.st/es-deprecation-7-monitoring-settings"; @@ -259,8 +261,8 @@ void monitoringExporterSetting(String suffix, String value) { String settingKey = "xpack.monitoring.exporters.test." + suffix; Settings settings = Settings.builder().put(settingKey, value).build(); final XPackLicenseState licenseState = new XPackLicenseState(() -> 0); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, licenseState) ); final String expectedUrl = "https://ela.st/es-deprecation-7-monitoring-settings"; @@ -284,8 +286,8 @@ void monitoringExporterGroupedSetting(String suffix, String value) { String subSettingKey = settingKey + ".subsetting"; Settings settings = Settings.builder().put(subSettingKey, value).build(); final XPackLicenseState licenseState = new XPackLicenseState(() -> 0); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, licenseState) ); final String expectedUrl = "https://ela.st/es-deprecation-7-monitoring-settings"; @@ -310,8 +312,8 @@ void monitoringExporterSecureSetting(String suffix, String value) { secureSettings.setString(settingKey, value); Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); final XPackLicenseState licenseState = new XPackLicenseState(() -> 0); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, licenseState) ); final String expectedUrl = "https://ela.st/es-deprecation-7-monitoring-settings"; @@ -457,8 +459,8 @@ public void testCheckMonitoringSettingCollectionInterval() { public void testExporterUseIngestPipelineSettings() { Settings settings = Settings.builder().put("xpack.monitoring.exporters.test.use_ingest", true).build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -483,8 +485,8 @@ public void testExporterPipelineMasterTimeoutSetting() { .put("xpack.monitoring.exporters.test.index.pipeline.master_timeout", TimeValue.timeValueSeconds(10)) .build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -508,8 +510,8 @@ public void testExporterPipelineMasterTimeoutSetting() { public void testExporterCreateLegacyTemplateSetting() { Settings settings = Settings.builder().put("xpack.monitoring.exporters.test.index.template.create_legacy_templates", true).build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -535,8 +537,8 @@ public void testScriptContextCacheSetting() { .put(ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), "use-context") .build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -564,8 +566,8 @@ public void testScriptContextCompilationsRateLimitSetting() { .put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(contexts.get(1)).getKey(), "456/7m") .build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -601,8 +603,8 @@ public void testImplicitScriptContextCacheSetting() { .put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(contexts.get(1)).getKey(), "2453") .build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -639,8 +641,8 @@ public void testScriptContextCacheSizeSetting() { .put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(contexts.get(1)).getKey(), 200) .build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -676,8 +678,8 @@ public void testScriptContextCacheExpirationSetting() { .put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(contexts.get(1)).getKey(), "2d") .build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -708,8 +710,8 @@ public void testScriptContextCacheExpirationSetting() { public void testEnforceDefaultTierPreferenceSetting() { Settings settings = Settings.builder().put(DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING.getKey(), randomBoolean()).build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -731,8 +733,8 @@ public void testEnforceDefaultTierPreferenceSetting() { } private List getDeprecationIssues(Settings settings, PluginsAndModules pluginsAndModules) { - final List issues = DeprecationChecks.filterChecks( - DeprecationChecks.NODE_SETTINGS_CHECKS, + final List issues = filterChecks( + NodeDeprecationChecks.SINGLE_NODE_CHECKS, c -> c.apply(settings, pluginsAndModules, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -799,8 +801,8 @@ public void testDynamicSettings() { } Metadata metadata = metadataBuilder.build(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); - final List issues = DeprecationChecks.filterChecks( - DeprecationChecks.NODE_SETTINGS_CHECKS, + final List issues = filterChecks( + NodeDeprecationChecks.SINGLE_NODE_CHECKS, c -> c.apply(nodettings, pluginsAndModules, clusterState, licenseState) ); @@ -832,4 +834,8 @@ public void testCheckNodeAttrData() { ); assertThat(issues, hasItem(expected)); } + + static List filterChecks(List checks, Function mapper) { + return checks.stream().map(mapper).filter(Objects::nonNull).toList(); + } } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationCheckerTests.java index 81c0d1c7dc918..69bb396a50a91 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationCheckerTests.java @@ -48,7 +48,7 @@ public void testCheckSourceModeInComponentTemplates() throws IOException { ) .build(); - Map> issuesByComponentTemplate = checker.check(clusterState, null); + Map> issuesByComponentTemplate = checker.check(clusterState); final DeprecationIssue expected = new DeprecationIssue( DeprecationIssue.Level.CRITICAL, SourceFieldMapper.DEPRECATION_WARNING, @@ -81,7 +81,7 @@ public void testCheckLegacyTiersInComponentTemplates() { ) .build(); - Map> issuesByComponentTemplate = checker.check(clusterState, null); + Map> issuesByComponentTemplate = checker.check(clusterState); final DeprecationIssue expected = new DeprecationIssue( DeprecationIssue.Level.WARNING, "Configuring tiers via filtered allocation is not recommended.", @@ -121,7 +121,7 @@ public void testCheckLegacyTierSettings() { ) .build(); - Map> issuesByComponentTemplate = checker.check(clusterState, null); + Map> issuesByComponentTemplate = checker.check(clusterState); final DeprecationIssue expected = new DeprecationIssue( DeprecationIssue.Level.WARNING, "Configuring tiers via filtered allocation is not recommended.", @@ -164,7 +164,7 @@ public void testComponentAndComposableTemplateWithSameName() { ) .build(); - Map> issuesByComponentTemplate = checker.check(clusterState, null); + Map> issuesByComponentTemplate = checker.check(clusterState); final DeprecationIssue expectedIndexTemplateIssue = new DeprecationIssue( DeprecationIssue.Level.WARNING, "Configuring tiers via filtered allocation is not recommended.", diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoActionTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoActionTests.java index 85fa375c09c5f..945068ba3a107 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoActionTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoActionTests.java @@ -6,22 +6,295 @@ */ package org.elasticsearch.xpack.deprecation; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; +import org.hamcrest.core.IsEqual; +import org.junit.Assert; +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.Response.RESERVED_NAMES; +import static org.elasticsearch.xpack.deprecation.DeprecationInfoActionResponseTests.createTestDeprecationIssue; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TransportDeprecationInfoActionTests extends ESTestCase { + public void testCheckAndCreateResponse() throws IOException { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all"); + mapping.field("enabled", false); + mapping.endObject().endObject(); + + Metadata metadata = Metadata.builder() + .put( + IndexMetadata.builder("test") + .putMapping(Strings.toString(mapping)) + .settings(settings(IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + ) + .build(); + + DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("test", new TransportAddress(TransportAddress.META_ADDRESS, 9300)); + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); + IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); + boolean clusterIssueFound = randomBoolean(); + boolean nodeIssueFound = randomBoolean(); + boolean indexIssueFound = randomBoolean(); + boolean dataStreamIssueFound = randomBoolean(); + boolean indexTemplateIssueFound = randomBoolean(); + boolean componentTemplateIssueFound = randomBoolean(); + boolean ilmPolicyIssueFound = randomBoolean(); + DeprecationIssue foundIssue = createTestDeprecationIssue(); + ClusterDeprecationChecker clusterDeprecationChecker = mock(ClusterDeprecationChecker.class); + when(clusterDeprecationChecker.check(any(), any())).thenReturn(clusterIssueFound ? List.of(foundIssue) : List.of()); + List resourceCheckers = List.of(createResourceChecker("index_settings", (cs, req) -> { + if (indexIssueFound) { + return Map.of("test", List.of(foundIssue)); + } + return Map.of(); + }), createResourceChecker("data_streams", (cs, req) -> { + if (dataStreamIssueFound) { + return Map.of("my-ds", List.of(foundIssue)); + } + return Map.of(); + }), createResourceChecker("templates", (cs, req) -> { + Map> issues = new HashMap<>(); + if (componentTemplateIssueFound) { + issues.put("my-component-template", List.of(foundIssue)); + } + if (indexTemplateIssueFound) { + issues.put("my-index-template", List.of(foundIssue)); + } + return issues; + }), createResourceChecker("ilm_policies", (cs, req) -> { + if (ilmPolicyIssueFound) { + return Map.of("my-policy", List.of(foundIssue)); + } + return Map.of(); + })); + + List nodeDeprecationIssues = nodeIssueFound ? List.of(foundIssue) : List.of(); + + DeprecationInfoAction.Request request = new DeprecationInfoAction.Request(randomTimeValue(), Strings.EMPTY_ARRAY); + TransportDeprecationInfoAction.PrecomputedData precomputedData = new TransportDeprecationInfoAction.PrecomputedData(); + precomputedData.setOnceTransformConfigs(List.of()); + precomputedData.setOncePluginIssues(Map.of()); + precomputedData.setOnceNodeSettingsIssues(nodeDeprecationIssues); + DeprecationInfoAction.Response response = TransportDeprecationInfoAction.checkAndCreateResponse( + state, + resolver, + request, + List.of(), + clusterDeprecationChecker, + resourceCheckers, + precomputedData + ); + + if (clusterIssueFound) { + assertThat(response.getClusterSettingsIssues(), IsEqual.equalTo(List.of(foundIssue))); + } else { + assertThat(response.getClusterSettingsIssues(), empty()); + } + + if (nodeIssueFound) { + assertThat(response.getNodeSettingsIssues(), IsEqual.equalTo(List.of(foundIssue))); + } else { + assertTrue(response.getNodeSettingsIssues().isEmpty()); + } + + if (indexIssueFound) { + assertThat(response.getIndexSettingsIssues(), IsEqual.equalTo(Map.of("test", List.of(foundIssue)))); + } else { + assertTrue(response.getIndexSettingsIssues().isEmpty()); + } + if (dataStreamIssueFound) { + assertThat(response.getDataStreamDeprecationIssues(), IsEqual.equalTo(Map.of("my-ds", List.of(foundIssue)))); + } else { + assertTrue(response.getDataStreamDeprecationIssues().isEmpty()); + } + if (ilmPolicyIssueFound) { + assertThat(response.getIlmPolicyDeprecationIssues(), IsEqual.equalTo(Map.of("my-policy", List.of(foundIssue)))); + } else { + assertTrue(response.getIlmPolicyDeprecationIssues().isEmpty()); + } + if (componentTemplateIssueFound == false && indexTemplateIssueFound == false) { + assertTrue(response.getTemplateDeprecationIssues().isEmpty()); + } else { + if (componentTemplateIssueFound) { + assertThat(response.getTemplateDeprecationIssues().get("my-component-template"), IsEqual.equalTo(List.of(foundIssue))); + } + if (indexTemplateIssueFound) { + assertThat(response.getTemplateDeprecationIssues().get("my-index-template"), IsEqual.equalTo(List.of(foundIssue))); + } + + } + } + + public void testRemoveSkippedSettings() { + Settings.Builder settingsBuilder = settings(IndexVersion.current()); + settingsBuilder.put("some.deprecated.property", "someValue1"); + settingsBuilder.put("some.other.bad.deprecated.property", "someValue2"); + settingsBuilder.put("some.undeprecated.property", "someValue3"); + settingsBuilder.putList("some.undeprecated.list.property", List.of("someValue4", "someValue5")); + Settings inputSettings = settingsBuilder.build(); + IndexMetadata dataStreamIndexMetadata = IndexMetadata.builder("ds-test-index-1") + .settings(inputSettings) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + ComponentTemplate componentTemplate = new ComponentTemplate(Template.builder().settings(inputSettings).build(), null, null); + ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() + .template(Template.builder().settings(inputSettings)) + .build(); + Metadata metadata = Metadata.builder() + .put(IndexMetadata.builder("test").settings(inputSettings).numberOfShards(1).numberOfReplicas(0)) + .put(dataStreamIndexMetadata, true) + .put(DataStream.builder("ds-test", List.of(dataStreamIndexMetadata.getIndex())).build()) + .indexTemplates( + Map.of( + "my-index-template", + indexTemplate, + "empty-template", + ComposableIndexTemplate.builder().indexPatterns(List.of("random")).build() + ) + ) + .componentTemplates(Map.of("my-component-template", componentTemplate)) + .persistentSettings(inputSettings) + .build(); + + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); + IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); + AtomicReference visibleClusterSettings = new AtomicReference<>(); + ClusterDeprecationChecker clusterDeprecationChecker = mock(ClusterDeprecationChecker.class); + when(clusterDeprecationChecker.check(any(), any())).thenAnswer(invocationOnMock -> { + ClusterState observedState = invocationOnMock.getArgument(0); + visibleClusterSettings.set(observedState.getMetadata().settings()); + return List.of(); + }); + AtomicReference visibleIndexSettings = new AtomicReference<>(); + AtomicReference visibleComponentTemplateSettings = new AtomicReference<>(); + AtomicReference visibleIndexTemplateSettings = new AtomicReference<>(); + AtomicInteger backingIndicesCount = new AtomicInteger(0); + List resourceCheckers = List.of(createResourceChecker("index_settings", (cs, req) -> { + for (String indexName : resolver.concreteIndexNames(cs, req)) { + visibleIndexSettings.set(cs.metadata().index(indexName).getSettings()); + } + return Map.of(); + }), createResourceChecker("data_streams", (cs, req) -> { + cs.metadata().dataStreams().values().forEach(ds -> backingIndicesCount.set(ds.getIndices().size())); + return Map.of(); + }), createResourceChecker("templates", (cs, req) -> { + cs.metadata() + .componentTemplates() + .values() + .forEach(template -> visibleComponentTemplateSettings.set(template.template().settings())); + cs.metadata().templatesV2().values().forEach(template -> { + if (template.template() != null && template.template().settings() != null) { + visibleIndexTemplateSettings.set(template.template().settings()); + } + }); + return Map.of(); + })); + TransportDeprecationInfoAction.PrecomputedData precomputedData = new TransportDeprecationInfoAction.PrecomputedData(); + precomputedData.setOnceTransformConfigs(List.of()); + precomputedData.setOncePluginIssues(Map.of()); + precomputedData.setOnceNodeSettingsIssues(List.of()); + DeprecationInfoAction.Request request = new DeprecationInfoAction.Request(randomTimeValue(), Strings.EMPTY_ARRAY); + TransportDeprecationInfoAction.checkAndCreateResponse( + state, + resolver, + request, + List.of("some.deprecated.property", "some.other.*.deprecated.property"), + clusterDeprecationChecker, + resourceCheckers, + precomputedData + ); + + settingsBuilder = settings(IndexVersion.current()); + settingsBuilder.put("some.undeprecated.property", "someValue3"); + settingsBuilder.putList("some.undeprecated.list.property", List.of("someValue4", "someValue5")); + + Settings expectedSettings = settingsBuilder.build(); + Settings resultClusterSettings = visibleClusterSettings.get(); + Assert.assertNotNull(resultClusterSettings); + Assert.assertEquals(expectedSettings, visibleClusterSettings.get()); + + Settings resultIndexSettings = visibleIndexSettings.get(); + Assert.assertNotNull(resultIndexSettings); + Assert.assertEquals("someValue3", resultIndexSettings.get("some.undeprecated.property")); + Assert.assertEquals(resultIndexSettings.getAsList("some.undeprecated.list.property"), List.of("someValue4", "someValue5")); + Assert.assertFalse(resultIndexSettings.hasValue("some.deprecated.property")); + Assert.assertFalse(resultIndexSettings.hasValue("some.other.bad.deprecated.property")); + + assertThat(backingIndicesCount.get(), IsEqual.equalTo(1)); + + Assert.assertNotNull(visibleComponentTemplateSettings.get()); + Assert.assertEquals(expectedSettings, visibleComponentTemplateSettings.get()); + Assert.assertNotNull(visibleIndexTemplateSettings.get()); + Assert.assertEquals(expectedSettings, visibleIndexTemplateSettings.get()); + } + + public void testCtorFailure() { + Map> indexNames = Stream.generate(() -> randomAlphaOfLength(10)) + .limit(10) + .collect(Collectors.toMap(Function.identity(), (_k) -> List.of())); + Map> dataStreamNames = Stream.generate(() -> randomAlphaOfLength(10)) + .limit(10) + .collect(Collectors.toMap(Function.identity(), (_k) -> List.of())); + Set shouldCauseFailure = new HashSet<>(RESERVED_NAMES); + for (int i = 0; i < randomIntBetween(1, 100); i++) { + Map> pluginSettingsIssues = randomSubsetOf(3, shouldCauseFailure).stream() + .collect(Collectors.toMap(Function.identity(), (_k) -> List.of())); + expectThrows( + ElasticsearchStatusException.class, + () -> new DeprecationInfoAction.Response( + List.of(), + List.of(), + Map.of("data_streams", dataStreamNames, "index_settings", indexNames), + pluginSettingsIssues + ) + ); + } + } + public void testPluginSettingIssues() { DeprecationChecker.Components components = new DeprecationChecker.Components(null, Settings.EMPTY, null); PlainActionFuture>> future = new PlainActionFuture<>(); @@ -65,6 +338,28 @@ public void testPluginSettingIssuesWithFailures() { assertThat(exception.getCause().getMessage(), containsString("boom")); } + private static ResourceDeprecationChecker createResourceChecker( + String name, + BiFunction>> check + ) { + return new ResourceDeprecationChecker() { + + @Override + public Map> check( + ClusterState clusterState, + DeprecationInfoAction.Request request, + TransportDeprecationInfoAction.PrecomputedData precomputedData + ) { + return check.apply(clusterState, request); + } + + @Override + public String getName() { + return name; + } + }; + } + private static class NamedChecker implements DeprecationChecker { private final String name; diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java index 80692efb7474a..a0a37f2bb52d1 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java @@ -61,7 +61,7 @@ public void testNodeOperation() { settingsBuilder.put("some.undeprecated.property", "someValue3"); settingsBuilder.putList("some.undeprecated.list.property", List.of("someValue4", "someValue5")); settingsBuilder.putList( - DeprecationChecks.SKIP_DEPRECATIONS_SETTING.getKey(), + TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING.getKey(), List.of("some.deprecated.property", "some.other.*.deprecated.property", "some.bad.dynamic.property") ); Settings nodeSettings = settingsBuilder.build(); @@ -73,7 +73,10 @@ public void testNodeOperation() { ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); ClusterService clusterService = Mockito.mock(ClusterService.class); when(clusterService.state()).thenReturn(clusterState); - ClusterSettings clusterSettings = new ClusterSettings(nodeSettings, Set.of(DeprecationChecks.SKIP_DEPRECATIONS_SETTING)); + ClusterSettings clusterSettings = new ClusterSettings( + nodeSettings, + Set.of(TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING) + ); when((clusterService.getClusterSettings())).thenReturn(clusterSettings); DiscoveryNode node = Mockito.mock(DiscoveryNode.class); when(node.getId()).thenReturn("mock-node"); @@ -98,7 +101,7 @@ public void testNodeOperation() { NodesDeprecationCheckAction.NodeRequest nodeRequest = null; AtomicReference visibleNodeSettings = new AtomicReference<>(); AtomicReference visibleClusterStateMetadataSettings = new AtomicReference<>(); - DeprecationChecks.NodeDeprecationCheck< + NodeDeprecationChecks.NodeDeprecationCheck< Settings, PluginsAndModules, ClusterState, @@ -109,7 +112,7 @@ public void testNodeOperation() { return null; }; java.util.List< - DeprecationChecks.NodeDeprecationCheck< + NodeDeprecationChecks.NodeDeprecationCheck< Settings, PluginsAndModules, ClusterState, @@ -120,7 +123,7 @@ public void testNodeOperation() { settingsBuilder.put("some.undeprecated.property", "someValue3"); settingsBuilder.putList("some.undeprecated.list.property", List.of("someValue4", "someValue5")); settingsBuilder.putList( - DeprecationChecks.SKIP_DEPRECATIONS_SETTING.getKey(), + TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING.getKey(), List.of("some.deprecated.property", "some.other.*.deprecated.property", "some.bad.dynamic.property") ); Settings expectedSettings = settingsBuilder.build(); @@ -131,7 +134,7 @@ public void testNodeOperation() { // Testing that the setting is dynamically updatable: Settings newSettings = Settings.builder() - .putList(DeprecationChecks.SKIP_DEPRECATIONS_SETTING.getKey(), List.of("some.undeprecated.property")) + .putList(TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING.getKey(), List.of("some.undeprecated.property")) .build(); clusterSettings.applySettings(newSettings); transportNodeDeprecationCheckAction.nodeOperation(nodeRequest, nodeSettingsChecks); @@ -141,7 +144,7 @@ public void testNodeOperation() { settingsBuilder.putList("some.undeprecated.list.property", List.of("someValue4", "someValue5")); // This is the node setting (since this is the node deprecation check), not the cluster setting: settingsBuilder.putList( - DeprecationChecks.SKIP_DEPRECATIONS_SETTING.getKey(), + TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING.getKey(), List.of("some.deprecated.property", "some.other.*.deprecated.property", "some.bad.dynamic.property") ); expectedSettings = settingsBuilder.build();