diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SourceModeRollingUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SourceModeRollingUpgradeIT.java index 98572de6618ee..8bf870edc5ec8 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SourceModeRollingUpgradeIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SourceModeRollingUpgradeIT.java @@ -18,7 +18,6 @@ import java.util.List; import java.util.Map; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class SourceModeRollingUpgradeIT extends AbstractRollingUpgradeTestCase { @@ -83,20 +82,10 @@ public void testConfigureStoredSourceWhenIndexIsCreatedLegacy() throws IOExcepti private void assertDeprecationWarningForTemplate(String templateName) throws IOException { var request = new Request("GET", "/_migration/deprecations"); var response = entityAsMap(client().performRequest(request)); - if (response.containsKey("templates")) { - // Check the newer version of the deprecation API that contains the templates section - Map issuesByTemplate = (Map) response.get("templates"); - assertThat(issuesByTemplate.containsKey(templateName), equalTo(true)); - var templateIssues = (List) issuesByTemplate.get(templateName); - assertThat(((Map) templateIssues.getFirst()).get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING)); - } else { - // Bwc version with 8.18 until https://github.com/elastic/elasticsearch/pull/120505/ gets backported, clean up after backport - var nodeSettings = (Map) ((List) response.get("node_settings")).getFirst(); - assertThat(nodeSettings.get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING)); - assertThat( - (String) nodeSettings.get("details"), - containsString(SourceFieldMapper.DEPRECATION_WARNING + " Affected component templates: [" + templateName + "]") - ); - } + assertThat(response.containsKey("templates"), equalTo(true)); + Map issuesByTemplate = (Map) response.get("templates"); + assertThat(issuesByTemplate.containsKey(templateName), equalTo(true)); + var templateIssues = (List) issuesByTemplate.get(templateName); + assertThat(((Map) templateIssues.getFirst()).get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING)); } } diff --git a/x-pack/plugin/deprecation/src/main/java/module-info.java b/x-pack/plugin/deprecation/src/main/java/module-info.java index f9a86839ad6f2..4c46205df4f0c 100644 --- a/x-pack/plugin/deprecation/src/main/java/module-info.java +++ b/x-pack/plugin/deprecation/src/main/java/module-info.java @@ -13,6 +13,7 @@ requires org.apache.logging.log4j; requires org.apache.logging.log4j.core; requires log4j2.ecs.layout; + requires org.apache.lucene.core; exports org.elasticsearch.xpack.deprecation to org.elasticsearch.server; exports org.elasticsearch.xpack.deprecation.logging to org.elasticsearch.server; diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecker.java new file mode 100644 index 0000000000000..cc21f0b2cd711 --- /dev/null +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecker.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.deprecation; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.common.TriConsumer; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; +import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Cluster-specific deprecation checks, this is used to populate the {@code cluster_settings} field + */ +public class ClusterDeprecationChecker { + + private static final Logger logger = LogManager.getLogger(ClusterDeprecationChecker.class); + private final List, List>> CHECKS = List.of( + this::checkTransformSettings + ); + private final NamedXContentRegistry xContentRegistry; + + ClusterDeprecationChecker(NamedXContentRegistry xContentRegistry) { + this.xContentRegistry = xContentRegistry; + } + + public List check(ClusterState clusterState, List transformConfigs) { + List allIssues = new ArrayList<>(); + CHECKS.forEach(check -> check.apply(clusterState, transformConfigs, allIssues)); + return allIssues; + } + + private void checkTransformSettings( + ClusterState clusterState, + List transformConfigs, + List allIssues + ) { + for (var config : transformConfigs) { + try { + allIssues.addAll(config.checkForDeprecations(xContentRegistry)); + } catch (IOException e) { + logger.warn("failed to check transformation settings for '" + config.getId() + "'", e); + } + } + } +} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java index 4844722ddcdd4..db8a3dc205e02 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java @@ -18,13 +18,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.BiFunction; import java.util.stream.Collectors; import static java.util.Map.entry; import static java.util.Map.ofEntries; -import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.filterChecks; /** * Checks the data streams for deprecation warnings. @@ -44,10 +44,24 @@ public DataStreamDeprecationChecker(IndexNameExpressionResolver indexNameExpress /** * @param clusterState The cluster state provided for the checker + * @param request not used yet in these checks + * @param precomputedData not used yet in these checks * @return the name of the data streams that have violated the checks with their respective warnings. */ @Override - public Map> check(ClusterState clusterState, DeprecationInfoAction.Request request) { + public Map> check( + ClusterState clusterState, + DeprecationInfoAction.Request request, + TransportDeprecationInfoAction.PrecomputedData precomputedData + ) { + return check(clusterState); + } + + /** + * @param clusterState The cluster state provided for the checker + * @return the name of the data streams that have violated the checks with their respective warnings. + */ + public Map> check(ClusterState clusterState) { List dataStreamNames = indexNameExpressionResolver.dataStreamNames( clusterState, IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN @@ -58,7 +72,10 @@ public Map> check(ClusterState clusterState, Depr Map> dataStreamIssues = new HashMap<>(); for (String dataStreamName : dataStreamNames) { DataStream dataStream = clusterState.metadata().dataStreams().get(dataStreamName); - List issuesForSingleDataStream = filterChecks(DATA_STREAM_CHECKS, c -> c.apply(dataStream, clusterState)); + List issuesForSingleDataStream = DATA_STREAM_CHECKS.stream() + .map(c -> c.apply(dataStream, clusterState)) + .filter(Objects::nonNull) + .toList(); if (issuesForSingleDataStream.isEmpty() == false) { dataStreamIssues.put(dataStreamName, issuesForSingleDataStream); } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java index 85b7c89e7cb85..2c8b95e378375 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java @@ -33,7 +33,7 @@ import java.util.function.Predicate; import java.util.function.Supplier; -import static org.elasticsearch.xpack.deprecation.DeprecationChecks.SKIP_DEPRECATIONS_SETTING; +import static org.elasticsearch.xpack.deprecation.TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING; import static org.elasticsearch.xpack.deprecation.logging.DeprecationIndexingComponent.DEPRECATION_INDEXING_FLUSH_INTERVAL; /** diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java deleted file mode 100644 index 039a75f51f030..0000000000000 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.deprecation; - -import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; - -import java.util.List; -import java.util.Objects; -import java.util.function.Function; -import java.util.stream.Collectors; - -/** - * Class containing all the cluster, node, and index deprecation checks that will be served - * by the {@link DeprecationInfoAction}. - */ -public class DeprecationChecks { - - public static final Setting> SKIP_DEPRECATIONS_SETTING = Setting.stringListSetting( - "deprecation.skip_deprecated_settings", - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - private DeprecationChecks() {} - - static List> CLUSTER_SETTINGS_CHECKS = List.of(); - - static final List< - NodeDeprecationCheck> NODE_SETTINGS_CHECKS = List - .of( - NodeDeprecationChecks::checkMultipleDataPaths, - NodeDeprecationChecks::checkDataPathsList, - NodeDeprecationChecks::checkSharedDataPathSetting, - NodeDeprecationChecks::checkReservedPrefixedRealmNames, - NodeDeprecationChecks::checkExporterUseIngestPipelineSettings, - NodeDeprecationChecks::checkExporterPipelineMasterTimeoutSetting, - NodeDeprecationChecks::checkExporterCreateLegacyTemplateSetting, - NodeDeprecationChecks::checkMonitoringSettingHistoryDuration, - NodeDeprecationChecks::checkMonitoringSettingHistoryDuration, - NodeDeprecationChecks::checkMonitoringSettingCollectIndexRecovery, - NodeDeprecationChecks::checkMonitoringSettingCollectIndices, - NodeDeprecationChecks::checkMonitoringSettingCollectCcrTimeout, - NodeDeprecationChecks::checkMonitoringSettingCollectEnrichStatsTimeout, - NodeDeprecationChecks::checkMonitoringSettingCollectIndexRecoveryStatsTimeout, - NodeDeprecationChecks::checkMonitoringSettingCollectIndexStatsTimeout, - NodeDeprecationChecks::checkMonitoringSettingCollectMlJobStatsTimeout, - NodeDeprecationChecks::checkMonitoringSettingCollectNodeStatsTimeout, - NodeDeprecationChecks::checkMonitoringSettingCollectClusterStatsTimeout, - NodeDeprecationChecks::checkMonitoringSettingExportersHost, - NodeDeprecationChecks::checkMonitoringSettingExportersBulkTimeout, - NodeDeprecationChecks::checkMonitoringSettingExportersConnectionTimeout, - NodeDeprecationChecks::checkMonitoringSettingExportersConnectionReadTimeout, - NodeDeprecationChecks::checkMonitoringSettingExportersAuthUsername, - NodeDeprecationChecks::checkMonitoringSettingExportersAuthPass, - NodeDeprecationChecks::checkMonitoringSettingExportersSSL, - NodeDeprecationChecks::checkMonitoringSettingExportersProxyBase, - NodeDeprecationChecks::checkMonitoringSettingExportersSniffEnabled, - NodeDeprecationChecks::checkMonitoringSettingExportersHeaders, - NodeDeprecationChecks::checkMonitoringSettingExportersTemplateTimeout, - NodeDeprecationChecks::checkMonitoringSettingExportersMasterTimeout, - NodeDeprecationChecks::checkMonitoringSettingExportersEnabled, - NodeDeprecationChecks::checkMonitoringSettingExportersType, - NodeDeprecationChecks::checkMonitoringSettingExportersAlertsEnabled, - NodeDeprecationChecks::checkMonitoringSettingExportersAlertsBlacklist, - NodeDeprecationChecks::checkMonitoringSettingExportersIndexNameTimeFormat, - NodeDeprecationChecks::checkMonitoringSettingDecommissionAlerts, - NodeDeprecationChecks::checkMonitoringSettingEsCollectionEnabled, - NodeDeprecationChecks::checkMonitoringSettingCollectionEnabled, - NodeDeprecationChecks::checkMonitoringSettingCollectionInterval, - NodeDeprecationChecks::checkScriptContextCache, - NodeDeprecationChecks::checkScriptContextCompilationsRateLimitSetting, - NodeDeprecationChecks::checkScriptContextCacheSizeSetting, - NodeDeprecationChecks::checkScriptContextCacheExpirationSetting, - NodeDeprecationChecks::checkEnforceDefaultTierPreferenceSetting, - NodeDeprecationChecks::checkLifecyleStepMasterTimeoutSetting, - NodeDeprecationChecks::checkEqlEnabledSetting, - NodeDeprecationChecks::checkNodeAttrData, - NodeDeprecationChecks::checkWatcherBulkConcurrentRequestsSetting, - NodeDeprecationChecks::checkTracingApmSettings - ); - - /** - * helper utility function to reduce repeat of running a specific {@link List} of checks. - * - * @param checks The functional checks to execute using the mapper function - * @param mapper The function that executes the lambda check with the appropriate arguments - * @param The signature of the check (BiFunction, Function, including the appropriate arguments) - * @return The list of {@link DeprecationIssue} that were found in the cluster - */ - static List filterChecks(List checks, Function mapper) { - return checks.stream().map(mapper).filter(Objects::nonNull).collect(Collectors.toList()); - } - - @FunctionalInterface - public interface NodeDeprecationCheck { - R apply(A first, B second, C third, D fourth); - } -} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java index 62d2a231face4..1fceb917ece53 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java @@ -12,41 +12,25 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.IndicesRequest; -import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeReadRequest; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.ComponentTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.regex.Regex; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.core.Tuple; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.transport.Transports; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; import static org.elasticsearch.action.ValidateActions.addValidationError; @@ -59,93 +43,6 @@ private DeprecationInfoAction() { super(NAME); } - /** - * helper utility function to reduce repeat of running a specific {@link Set} of checks. - * - * @param checks The functional checks to execute using the mapper function - * @param mapper The function that executes the lambda check with the appropriate arguments - * @param The signature of the check (BiFunction, Function, including the appropriate arguments) - * @return The list of {@link DeprecationIssue} that were found in the cluster - */ - public static List filterChecks(List checks, Function mapper) { - return checks.stream().map(mapper).filter(Objects::nonNull).collect(Collectors.toList()); - } - - /** - * This method rolls up DeprecationIssues that are identical but on different nodes. It also roles up DeprecationIssues that are - * identical (and on different nodes) except that they differ in the removable settings listed in their meta object. We roll these up - * by taking the intersection of all removable settings in otherwise identical DeprecationIssues. That way we don't claim that a - * setting can be automatically removed if any node has it in its elasticsearch.yml. - * @param response - * @return - */ - private static List mergeNodeIssues(NodesDeprecationCheckResponse response) { - // A collection whose values are lists of DeprecationIssues that differ only by meta values (if that): - Collection>> issuesToMerge = getDeprecationIssuesThatDifferOnlyByMeta(response.getNodes()); - // A map of DeprecationIssues (containing only the intersection of removable settings) to the nodes they are seen on - Map> issueToListOfNodesMap = getMergedIssuesToNodesMap(issuesToMerge); - - return issueToListOfNodesMap.entrySet().stream().map(entry -> { - DeprecationIssue issue = entry.getKey(); - String details = issue.getDetails() != null ? issue.getDetails() + " " : ""; - return new DeprecationIssue( - issue.getLevel(), - issue.getMessage(), - issue.getUrl(), - details + "(nodes impacted: " + entry.getValue() + ")", - issue.isResolveDuringRollingUpgrade(), - issue.getMeta() - ); - }).collect(Collectors.toList()); - } - - /* - * This method pulls all the DeprecationIssues from the given nodeResponses, and buckets them into lists of DeprecationIssues that - * differ at most by meta values (if that). The returned tuples also contain the node name the deprecation issue was found on. If all - * nodes in the cluster were configured identically then all tuples in a list will differ only by the node name. - */ - private static Collection>> getDeprecationIssuesThatDifferOnlyByMeta( - List nodeResponses - ) { - Map>> issuesToMerge = new HashMap<>(); - for (NodesDeprecationCheckAction.NodeResponse resp : nodeResponses) { - for (DeprecationIssue issue : resp.getDeprecationIssues()) { - issuesToMerge.computeIfAbsent( - new DeprecationIssue( - issue.getLevel(), - issue.getMessage(), - issue.getUrl(), - issue.getDetails(), - issue.isResolveDuringRollingUpgrade(), - null // Intentionally removing meta from the key so that it's not taken into account for equality - ), - (key) -> new ArrayList<>() - ).add(new Tuple<>(issue, resp.getNode().getName())); - } - } - return issuesToMerge.values(); - } - - /* - * At this point we have one DeprecationIssue per node for a given deprecation. This method rolls them up into a single DeprecationIssue - * with a list of nodes that they appear on. If two DeprecationIssues on two different nodes differ only by the set of removable - * settings (i.e. they have different elasticsearch.yml configurations) then this method takes the intersection of those settings when - * it rolls them up. - */ - private static Map> getMergedIssuesToNodesMap( - Collection>> issuesToMerge - ) { - Map> issueToListOfNodesMap = new HashMap<>(); - for (List> similarIssues : issuesToMerge) { - DeprecationIssue leastCommonDenominator = DeprecationIssue.getIntersectionOfRemovableSettings( - similarIssues.stream().map(Tuple::v1).toList() - ); - issueToListOfNodesMap.computeIfAbsent(leastCommonDenominator, (key) -> new ArrayList<>()) - .addAll(similarIssues.stream().map(Tuple::v2).toList()); - } - return issueToListOfNodesMap; - } - public static class Response extends ActionResponse implements ToXContentObject { static final Set RESERVED_NAMES = Set.of( "cluster_settings", @@ -289,143 +186,6 @@ public int hashCode() { return Objects.hash(clusterSettingsIssues, nodeSettingsIssues, resourceDeprecationIssues, pluginSettingsIssues); } - /** - * This is the function that does the bulk of the logic of taking the appropriate ES dependencies - * like {@link NodeInfo}, {@link ClusterState}. Alongside these objects and the list of deprecation checks, - * this function will run through all the checks and build out the final list of issues that exist in the - * cluster. - * - * @param state The cluster state - * @param indexNameExpressionResolver Used to resolve indices into their concrete names - * @param request The originating request containing the index expressions to evaluate - * @param nodeDeprecationResponse The response containing the deprecation issues found on each node - * @param clusterSettingsChecks The list of cluster-level checks - * @param pluginSettingIssues this map gets modified to move transform deprecation issues into cluster_settings - * @param skipTheseDeprecatedSettings the settings that will be removed from cluster metadata and the index metadata of all the - * indexes specified by indexNames - * @param resourceDeprecationCheckers these are checkers that take as input the cluster state and return a map from resource type - * to issues grouped by the resource name. - * @return The list of deprecation issues found in the cluster - */ - public static DeprecationInfoAction.Response from( - ClusterState state, - IndexNameExpressionResolver indexNameExpressionResolver, - Request request, - NodesDeprecationCheckResponse nodeDeprecationResponse, - List> clusterSettingsChecks, - Map> pluginSettingIssues, - List skipTheseDeprecatedSettings, - List resourceDeprecationCheckers - ) { - assert Transports.assertNotTransportThread("walking mappings in indexSettingsChecks is expensive"); - // Allow system index access here to prevent deprecation warnings when we call this API - String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames(state, request); - ClusterState stateWithSkippedSettingsRemoved = removeSkippedSettings(state, concreteIndexNames, skipTheseDeprecatedSettings); - List clusterSettingsIssues = filterChecks( - clusterSettingsChecks, - (c) -> c.apply(stateWithSkippedSettingsRemoved) - ); - List nodeSettingsIssues = mergeNodeIssues(nodeDeprecationResponse); - - Map>> resourceDeprecationIssues = new HashMap<>(); - for (ResourceDeprecationChecker resourceDeprecationChecker : resourceDeprecationCheckers) { - Map> issues = resourceDeprecationChecker.check(stateWithSkippedSettingsRemoved, request); - if (issues.isEmpty() == false) { - resourceDeprecationIssues.put(resourceDeprecationChecker.getName(), issues); - } - } - - // WORKAROUND: move transform deprecation issues into cluster_settings - List transformDeprecations = pluginSettingIssues.remove( - TransformDeprecationChecker.TRANSFORM_DEPRECATION_KEY - ); - if (transformDeprecations != null) { - clusterSettingsIssues.addAll(transformDeprecations); - } - - return new DeprecationInfoAction.Response( - clusterSettingsIssues, - nodeSettingsIssues, - resourceDeprecationIssues, - pluginSettingIssues - ); - } - } - - /** - * Removes the skipped settings from the selected indices and the component and index templates. - * @param state The cluster state to modify - * @param indexNames The names of the indexes whose settings need to be filtered - * @param skipTheseDeprecatedSettings The settings that will be removed from cluster metadata and the index metadata of all the - * indexes specified by indexNames - * @return A modified cluster state with the given settings removed - */ - private static ClusterState removeSkippedSettings(ClusterState state, String[] indexNames, List skipTheseDeprecatedSettings) { - // Short-circuit, no need to reconstruct the cluster state if there are no settings to remove - if (skipTheseDeprecatedSettings == null || skipTheseDeprecatedSettings.isEmpty()) { - return state; - } - ClusterState.Builder clusterStateBuilder = new ClusterState.Builder(state); - Metadata.Builder metadataBuilder = Metadata.builder(state.metadata()); - metadataBuilder.transientSettings( - metadataBuilder.transientSettings().filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false) - ); - metadataBuilder.persistentSettings( - metadataBuilder.persistentSettings().filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false) - ); - Map indicesBuilder = new HashMap<>(state.getMetadata().indices()); - for (String indexName : indexNames) { - IndexMetadata indexMetadata = state.getMetadata().index(indexName); - IndexMetadata.Builder filteredIndexMetadataBuilder = new IndexMetadata.Builder(indexMetadata); - Settings filteredSettings = indexMetadata.getSettings() - .filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false); - filteredIndexMetadataBuilder.settings(filteredSettings); - indicesBuilder.put(indexName, filteredIndexMetadataBuilder.build()); - } - metadataBuilder.componentTemplates(state.metadata().componentTemplates().entrySet().stream().map(entry -> { - String templateName = entry.getKey(); - ComponentTemplate componentTemplate = entry.getValue(); - Template template = componentTemplate.template(); - if (template.settings() == null || template.settings().isEmpty()) { - return Tuple.tuple(templateName, componentTemplate); - } - return Tuple.tuple( - templateName, - new ComponentTemplate( - Template.builder(template) - .settings(template.settings().filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false)) - .build(), - componentTemplate.version(), - componentTemplate.metadata(), - componentTemplate.deprecated() - ) - ); - }).collect(Collectors.toMap(Tuple::v1, Tuple::v2))); - metadataBuilder.indexTemplates(state.metadata().templatesV2().entrySet().stream().map(entry -> { - String templateName = entry.getKey(); - ComposableIndexTemplate indexTemplate = entry.getValue(); - Template template = indexTemplate.template(); - if (template == null || template.settings() == null || template.settings().isEmpty()) { - return Tuple.tuple(templateName, indexTemplate); - } - return Tuple.tuple( - templateName, - indexTemplate.toBuilder() - .template( - Template.builder(indexTemplate.template()) - .settings( - indexTemplate.template() - .settings() - .filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false) - ) - ) - .build() - ); - }).collect(Collectors.toMap(Tuple::v1, Tuple::v2))); - - metadataBuilder.indices(indicesBuilder); - clusterStateBuilder.metadata(metadataBuilder); - return clusterStateBuilder.build(); } public static class Request extends MasterNodeReadRequest implements IndicesRequest.Replaceable { diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationChecker.java index 6d7f860f645f1..f96fae6343b9f 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationChecker.java @@ -19,9 +19,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.function.Function; -import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.filterChecks; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_COMMON_DETAIL; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_HELP_URL; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_MESSAGE; @@ -33,17 +33,28 @@ public class IlmPolicyDeprecationChecker implements ResourceDeprecationChecker { public static final String NAME = "ilm_policies"; - private static final List> CHECKS = List.of( - IlmPolicyDeprecationChecker::checkLegacyTiers, - IlmPolicyDeprecationChecker::checkFrozenAction - ); + private final List> checks = List.of(this::checkLegacyTiers, this::checkFrozenAction); /** * @param clusterState The cluster state provided for the checker + * @param request not used yet in these checks + * @param precomputedData not used yet in these checks * @return the name of the data streams that have violated the checks with their respective warnings. */ @Override - public Map> check(ClusterState clusterState, DeprecationInfoAction.Request request) { + public Map> check( + ClusterState clusterState, + DeprecationInfoAction.Request request, + TransportDeprecationInfoAction.PrecomputedData precomputedData + ) { + return check(clusterState); + } + + /** + * @param clusterState The cluster state provided for the checker + * @return the name of the data streams that have violated the checks with their respective warnings. + */ + Map> check(ClusterState clusterState) { IndexLifecycleMetadata lifecycleMetadata = clusterState.metadata().custom(IndexLifecycleMetadata.TYPE); if (lifecycleMetadata == null || lifecycleMetadata.getPolicyMetadatas().isEmpty()) { return Map.of(); @@ -53,7 +64,10 @@ public Map> check(ClusterState clusterState, Depr String name = entry.getKey(); LifecyclePolicyMetadata policyMetadata = entry.getValue(); - List issuesForSinglePolicy = filterChecks(CHECKS, c -> c.apply(policyMetadata.getPolicy())); + List issuesForSinglePolicy = checks.stream() + .map(c -> c.apply(policyMetadata.getPolicy())) + .filter(Objects::nonNull) + .toList(); if (issuesForSinglePolicy.isEmpty() == false) { issues.put(name, issuesForSinglePolicy); } @@ -61,7 +75,7 @@ public Map> check(ClusterState clusterState, Depr return issues.isEmpty() ? Map.of() : issues; } - static DeprecationIssue checkLegacyTiers(LifecyclePolicy policy) { + private DeprecationIssue checkLegacyTiers(LifecyclePolicy policy) { for (Phase phase : policy.getPhases().values()) { AllocateAction allocateAction = (AllocateAction) phase.getActions().get(AllocateAction.NAME); if (allocateAction != null) { @@ -82,7 +96,7 @@ static DeprecationIssue checkLegacyTiers(LifecyclePolicy policy) { return null; } - static DeprecationIssue checkFrozenAction(LifecyclePolicy policy) { + private DeprecationIssue checkFrozenAction(LifecyclePolicy policy) { for (Phase phase : policy.getPhases().values()) { if (phase.getActions().containsKey(FreezeAction.NAME)) { return new DeprecationIssue( diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java index 778e4d176ca0f..844988c7a4896 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java @@ -10,6 +10,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.LegacyFormatNames; import org.elasticsearch.index.IndexModule; @@ -17,17 +18,19 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.xpack.core.deprecation.DeprecatedIndexPredicate; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; +import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Function; +import java.util.stream.Collectors; -import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.filterChecks; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_COMMON_DETAIL; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_HELP_URL; @@ -39,20 +42,35 @@ public class IndexDeprecationChecker implements ResourceDeprecationChecker { public static final String NAME = "index_settings"; private final IndexNameExpressionResolver indexNameExpressionResolver; - private final Map> indexToTransformIds; + private final List>, DeprecationIssue>> checks = List.of( + this::oldIndicesCheck, + this::ignoredOldIndicesCheck, + this::translogRetentionSettingCheck, + this::checkIndexDataPath, + this::storeTypeSettingCheck, + this::deprecatedCamelCasePattern, + this::legacyRoutingSettingCheck + ); - public IndexDeprecationChecker(IndexNameExpressionResolver indexNameExpressionResolver, Map> indexToTransformIds) { + public IndexDeprecationChecker(IndexNameExpressionResolver indexNameExpressionResolver) { this.indexNameExpressionResolver = indexNameExpressionResolver; - this.indexToTransformIds = indexToTransformIds; } @Override - public Map> check(ClusterState clusterState, DeprecationInfoAction.Request request) { + public Map> check( + ClusterState clusterState, + DeprecationInfoAction.Request request, + TransportDeprecationInfoAction.PrecomputedData precomputedData + ) { Map> indexSettingsIssues = new HashMap<>(); String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames(clusterState, request); + Map> indexToTransformIds = indexToTransformIds(precomputedData.transformConfigs()); for (String concreteIndex : concreteIndexNames) { IndexMetadata indexMetadata = clusterState.getMetadata().index(concreteIndex); - List singleIndexIssues = filterChecks(indexSettingsChecks(), c -> c.apply(indexMetadata, clusterState)); + List singleIndexIssues = checks.stream() + .map(c -> c.apply(indexMetadata, clusterState, indexToTransformIds)) + .filter(Objects::nonNull) + .toList(); if (singleIndexIssues.isEmpty() == false) { indexSettingsIssues.put(concreteIndex, singleIndexIssues); } @@ -63,24 +81,16 @@ public Map> check(ClusterState clusterState, Depr return indexSettingsIssues; } - private List> indexSettingsChecks() { - return List.of( - this::oldIndicesCheck, - this::ignoredOldIndicesCheck, - IndexDeprecationChecker::translogRetentionSettingCheck, - IndexDeprecationChecker::checkIndexDataPath, - IndexDeprecationChecker::storeTypeSettingCheck, - IndexDeprecationChecker::deprecatedCamelCasePattern, - IndexDeprecationChecker::legacyRoutingSettingCheck - ); - } - @Override public String getName() { return NAME; } - private DeprecationIssue oldIndicesCheck(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue oldIndicesCheck( + IndexMetadata indexMetadata, + ClusterState clusterState, + Map> indexToTransformIds + ) { // TODO: this check needs to be revised. It's trivially true right now. IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks @@ -91,13 +101,13 @@ private DeprecationIssue oldIndicesCheck(IndexMetadata indexMetadata, ClusterSta "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html", "This index has version: " + currentCompatibilityVersion.toReleaseVersion(), false, - meta(indexMetadata) + meta(indexMetadata, indexToTransformIds) ); } return null; } - private Map meta(IndexMetadata indexMetadata) { + private Map meta(IndexMetadata indexMetadata, Map> indexToTransformIds) { var transforms = indexToTransformIds.getOrDefault(indexMetadata.getIndex().getName(), List.of()); if (transforms.isEmpty()) { return Map.of("reindex_required", true); @@ -106,7 +116,11 @@ private Map meta(IndexMetadata indexMetadata) { } } - private DeprecationIssue ignoredOldIndicesCheck(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue ignoredOldIndicesCheck( + IndexMetadata indexMetadata, + ClusterState clusterState, + Map> indexToTransformIds + ) { IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, true) && isNotDataStreamIndex(indexMetadata, clusterState)) { @@ -118,17 +132,21 @@ private DeprecationIssue ignoredOldIndicesCheck(IndexMetadata indexMetadata, Clu + currentCompatibilityVersion.toReleaseVersion() + " and will be supported as read-only in 9.0", false, - meta(indexMetadata) + meta(indexMetadata, indexToTransformIds) ); } return null; } - private static boolean isNotDataStreamIndex(IndexMetadata indexMetadata, ClusterState clusterState) { + private boolean isNotDataStreamIndex(IndexMetadata indexMetadata, ClusterState clusterState) { return clusterState.metadata().findDataStreams(indexMetadata.getIndex().getName()).isEmpty(); } - private static DeprecationIssue translogRetentionSettingCheck(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue translogRetentionSettingCheck( + IndexMetadata indexMetadata, + ClusterState clusterState, + Map> ignored + ) { final boolean softDeletesEnabled = IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexMetadata.getSettings()); if (softDeletesEnabled) { if (IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.exists(indexMetadata.getSettings()) @@ -155,7 +173,7 @@ private static DeprecationIssue translogRetentionSettingCheck(IndexMetadata inde return null; } - private static DeprecationIssue checkIndexDataPath(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue checkIndexDataPath(IndexMetadata indexMetadata, ClusterState clusterState, Map> ignored) { if (IndexMetadata.INDEX_DATA_PATH_SETTING.exists(indexMetadata.getSettings())) { final String message = String.format( Locale.ROOT, @@ -170,7 +188,11 @@ private static DeprecationIssue checkIndexDataPath(IndexMetadata indexMetadata, return null; } - private static DeprecationIssue storeTypeSettingCheck(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue storeTypeSettingCheck( + IndexMetadata indexMetadata, + ClusterState clusterState, + Map> ignored + ) { final String storeType = IndexModule.INDEX_STORE_TYPE_SETTING.get(indexMetadata.getSettings()); if (IndexModule.Type.SIMPLEFS.match(storeType)) { return new DeprecationIssue( @@ -187,7 +209,11 @@ private static DeprecationIssue storeTypeSettingCheck(IndexMetadata indexMetadat return null; } - private static DeprecationIssue legacyRoutingSettingCheck(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue legacyRoutingSettingCheck( + IndexMetadata indexMetadata, + ClusterState clusterState, + Map> ignored + ) { List deprecatedSettings = LegacyTiersDetection.getDeprecatedFilteredAllocationSettings(indexMetadata.getSettings()); if (deprecatedSettings.isEmpty()) { return null; @@ -203,7 +229,7 @@ private static DeprecationIssue legacyRoutingSettingCheck(IndexMetadata indexMet ); } - private static void fieldLevelMappingIssue(IndexMetadata indexMetadata, BiConsumer> checker) { + private void fieldLevelMappingIssue(IndexMetadata indexMetadata, BiConsumer> checker) { if (indexMetadata.mapping() != null) { Map sourceAsMap = indexMetadata.mapping().sourceAsMap(); checker.accept(indexMetadata.mapping(), sourceAsMap); @@ -221,7 +247,7 @@ private static void fieldLevelMappingIssue(IndexMetadata indexMetadata, BiConsum * @return a list of issues found in fields */ @SuppressWarnings("unchecked") - private static List findInPropertiesRecursively( + private List findInPropertiesRecursively( String type, Map parentMap, Function, Boolean> predicate, @@ -275,7 +301,11 @@ private static List findInPropertiesRecursively( return issues; } - private static DeprecationIssue deprecatedCamelCasePattern(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue deprecatedCamelCasePattern( + IndexMetadata indexMetadata, + ClusterState clusterState, + Map> ignored + ) { List fields = new ArrayList<>(); fieldLevelMappingIssue( indexMetadata, @@ -283,8 +313,8 @@ private static DeprecationIssue deprecatedCamelCasePattern(IndexMetadata indexMe findInPropertiesRecursively( mappingMetadata.type(), sourceAsMap, - IndexDeprecationChecker::isDateFieldWithCamelCasePattern, - IndexDeprecationChecker::changeFormatToSnakeCase, + this::isDateFieldWithCamelCasePattern, + this::changeFormatToSnakeCase, "", "" ) @@ -305,7 +335,7 @@ private static DeprecationIssue deprecatedCamelCasePattern(IndexMetadata indexMe return null; } - private static boolean isDateFieldWithCamelCasePattern(Map property) { + private boolean isDateFieldWithCamelCasePattern(Map property) { if ("date".equals(property.get("type")) && property.containsKey("format")) { String[] patterns = DateFormatter.splitCombinedPatterns((String) property.get("format")); for (String pattern : patterns) { @@ -316,7 +346,7 @@ private static boolean isDateFieldWithCamelCasePattern(Map property) { return false; } - private static String changeFormatToSnakeCase(String type, Map.Entry entry) { + private String changeFormatToSnakeCase(String type, Map.Entry entry) { Map value = (Map) entry.getValue(); final String formatFieldValue = (String) value.get("format"); String[] patterns = DateFormatter.splitCombinedPatterns(formatFieldValue); @@ -332,4 +362,14 @@ private static String changeFormatToSnakeCase(String type, Map.Entry entry sb.deleteCharAt(sb.length() - 1); return sb.toString(); } + + private Map> indexToTransformIds(List transformConfigs) { + return transformConfigs.stream() + .collect( + Collectors.groupingBy( + config -> config.getDestination().getIndex(), + Collectors.mapping(TransformConfig::getId, Collectors.toList()) + ) + ); + } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/LegacyIndexTemplateDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/LegacyIndexTemplateDeprecationChecker.java deleted file mode 100644 index f7aba6491dfd2..0000000000000 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/LegacyIndexTemplateDeprecationChecker.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.deprecation; - -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; -import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Function; - -import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.filterChecks; -import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_COMMON_DETAIL; -import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_HELP_URL; -import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_MESSAGE; - -/** - * Checks the legacy index templates for deprecation warnings. - */ -public class LegacyIndexTemplateDeprecationChecker implements ResourceDeprecationChecker { - - public static final String NAME = "legacy_templates"; - private static final List> CHECKS = List.of( - LegacyIndexTemplateDeprecationChecker::checkIndexTemplates - ); - - /** - * @param clusterState The cluster state provided for the checker - * @return the name of the data streams that have violated the checks with their respective warnings. - */ - @Override - public Map> check(ClusterState clusterState, DeprecationInfoAction.Request request) { - var templates = clusterState.metadata().templates().entrySet(); - if (templates.isEmpty()) { - return Map.of(); - } - Map> issues = new HashMap<>(); - for (Map.Entry entry : templates) { - String name = entry.getKey(); - IndexTemplateMetadata template = entry.getValue(); - - List issuesForSingleIndexTemplate = filterChecks(CHECKS, c -> c.apply(template)); - if (issuesForSingleIndexTemplate.isEmpty() == false) { - issues.put(name, issuesForSingleIndexTemplate); - } - } - return issues.isEmpty() ? Map.of() : issues; - } - - static DeprecationIssue checkIndexTemplates(IndexTemplateMetadata indexTemplateMetadata) { - List deprecatedSettings = LegacyTiersDetection.getDeprecatedFilteredAllocationSettings(indexTemplateMetadata.settings()); - if (deprecatedSettings.isEmpty()) { - return null; - } - return new DeprecationIssue( - DeprecationIssue.Level.WARNING, - DEPRECATION_MESSAGE, - DEPRECATION_HELP_URL, - "One or more of your legacy index templates is configured with 'index.routing.allocation.*.data' settings. " - + DEPRECATION_COMMON_DETAIL, - false, - DeprecationIssue.createMetaMapForRemovableSettings(deprecatedSettings) - ); - } - - @Override - public String getName() { - return NAME; - } -} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecker.java new file mode 100644 index 0000000000000..a2e9ed12a2298 --- /dev/null +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecker.java @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.deprecation; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.ThreadedActionListener; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Retrieves the individual node checks and reduces them to a list of deprecation warnings + */ +public class NodeDeprecationChecker { + + private static final Logger logger = LogManager.getLogger(NodeDeprecationChecker.class); + private final ThreadPool threadPool; + + public NodeDeprecationChecker(ThreadPool threadPool) { + this.threadPool = threadPool; + } + + public void check(Client client, ActionListener> listener) { + NodesDeprecationCheckRequest nodeDepReq = new NodesDeprecationCheckRequest("_all"); + ClientHelper.executeAsyncWithOrigin( + client, + ClientHelper.DEPRECATION_ORIGIN, + NodesDeprecationCheckAction.INSTANCE, + nodeDepReq, + new ThreadedActionListener<>(threadPool.generic(), listener.delegateFailureAndWrap((l, response) -> { + if (response.hasFailures()) { + List failedNodeIds = response.failures() + .stream() + .map(failure -> failure.nodeId() + ": " + failure.getMessage()) + .collect(Collectors.toList()); + logger.warn("nodes failed to run deprecation checks: {}", failedNodeIds); + for (FailedNodeException failure : response.failures()) { + logger.debug("node {} failed to run deprecation checks: {}", failure.nodeId(), failure); + } + } + l.onResponse(reduceToDeprecationIssues(response)); + })) + ); + } + + /** + * This method rolls up DeprecationIssues that are identical but on different nodes. It also rolls up DeprecationIssues that are + * identical (and on different nodes) except that they differ in the removable settings listed in their meta object. We roll these up + * by taking the intersection of all removable settings in otherwise identical DeprecationIssues. That way we don't claim that a + * setting can be automatically removed if any node has it in its elasticsearch.yml. + * @param response the response that contains the deprecation issues of single nodes + * @return a list of deprecation issues grouped accordingly. + */ + static List reduceToDeprecationIssues(NodesDeprecationCheckResponse response) { + // A collection whose values are lists of DeprecationIssues that differ only by meta values (if that): + Collection>> issuesToMerge = getDeprecationIssuesThatDifferOnlyByMeta(response.getNodes()); + // A map of DeprecationIssues (containing only the intersection of removable settings) to the nodes they are seen on + Map> issueToListOfNodesMap = getMergedIssuesToNodesMap(issuesToMerge); + + return issueToListOfNodesMap.entrySet().stream().map(entry -> { + DeprecationIssue issue = entry.getKey(); + String details = issue.getDetails() != null ? issue.getDetails() + " " : ""; + return new DeprecationIssue( + issue.getLevel(), + issue.getMessage(), + issue.getUrl(), + details + "(nodes impacted: " + entry.getValue() + ")", + issue.isResolveDuringRollingUpgrade(), + issue.getMeta() + ); + }).collect(Collectors.toList()); + } + + /* + * This method pulls all the DeprecationIssues from the given nodeResponses, and buckets them into lists of DeprecationIssues that + * differ at most by meta values (if that). The returned tuples also contain the node name the deprecation issue was found on. If all + * nodes in the cluster were configured identically then all tuples in a list will differ only by the node name. + */ + private static Collection>> getDeprecationIssuesThatDifferOnlyByMeta( + List nodeResponses + ) { + Map>> issuesToMerge = new HashMap<>(); + for (NodesDeprecationCheckAction.NodeResponse resp : nodeResponses) { + for (DeprecationIssue issue : resp.getDeprecationIssues()) { + issuesToMerge.computeIfAbsent( + new DeprecationIssue( + issue.getLevel(), + issue.getMessage(), + issue.getUrl(), + issue.getDetails(), + issue.isResolveDuringRollingUpgrade(), + null // Intentionally removing meta from the key so that it's not taken into account for equality + ), + (key) -> new ArrayList<>() + ).add(new Tuple<>(issue, resp.getNode().getName())); + } + } + return issuesToMerge.values(); + } + + /* + * At this point we have one DeprecationIssue per node for a given deprecation. This method rolls them up into a single DeprecationIssue + * with a list of nodes that they appear on. If two DeprecationIssues on two different nodes differ only by the set of removable + * settings (i.e. they have different elasticsearch.yml configurations) then this method takes the intersection of those settings when + * it rolls them up. + */ + private static Map> getMergedIssuesToNodesMap( + Collection>> issuesToMerge + ) { + Map> issueToListOfNodesMap = new HashMap<>(); + for (List> similarIssues : issuesToMerge) { + DeprecationIssue leastCommonDenominator = DeprecationIssue.getIntersectionOfRemovableSettings( + similarIssues.stream().map(Tuple::v1).toList() + ); + issueToListOfNodesMap.computeIfAbsent(leastCommonDenominator, (key) -> new ArrayList<>()) + .addAll(similarIssues.stream().map(Tuple::v2).toList()); + } + return issueToListOfNodesMap; + } + +} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java index b6fff5a82f0cd..2f476d111f4ba 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java @@ -41,6 +41,60 @@ public class NodeDeprecationChecks { + // Visible for testing + static final List< + NodeDeprecationCheck> SINGLE_NODE_CHECKS = List.of( + NodeDeprecationChecks::checkMultipleDataPaths, + NodeDeprecationChecks::checkDataPathsList, + NodeDeprecationChecks::checkSharedDataPathSetting, + NodeDeprecationChecks::checkReservedPrefixedRealmNames, + NodeDeprecationChecks::checkExporterUseIngestPipelineSettings, + NodeDeprecationChecks::checkExporterPipelineMasterTimeoutSetting, + NodeDeprecationChecks::checkExporterCreateLegacyTemplateSetting, + NodeDeprecationChecks::checkMonitoringSettingHistoryDuration, + NodeDeprecationChecks::checkMonitoringSettingHistoryDuration, + NodeDeprecationChecks::checkMonitoringSettingCollectIndexRecovery, + NodeDeprecationChecks::checkMonitoringSettingCollectIndices, + NodeDeprecationChecks::checkMonitoringSettingCollectCcrTimeout, + NodeDeprecationChecks::checkMonitoringSettingCollectEnrichStatsTimeout, + NodeDeprecationChecks::checkMonitoringSettingCollectIndexRecoveryStatsTimeout, + NodeDeprecationChecks::checkMonitoringSettingCollectIndexStatsTimeout, + NodeDeprecationChecks::checkMonitoringSettingCollectMlJobStatsTimeout, + NodeDeprecationChecks::checkMonitoringSettingCollectNodeStatsTimeout, + NodeDeprecationChecks::checkMonitoringSettingCollectClusterStatsTimeout, + NodeDeprecationChecks::checkMonitoringSettingExportersHost, + NodeDeprecationChecks::checkMonitoringSettingExportersBulkTimeout, + NodeDeprecationChecks::checkMonitoringSettingExportersConnectionTimeout, + NodeDeprecationChecks::checkMonitoringSettingExportersConnectionReadTimeout, + NodeDeprecationChecks::checkMonitoringSettingExportersAuthUsername, + NodeDeprecationChecks::checkMonitoringSettingExportersAuthPass, + NodeDeprecationChecks::checkMonitoringSettingExportersSSL, + NodeDeprecationChecks::checkMonitoringSettingExportersProxyBase, + NodeDeprecationChecks::checkMonitoringSettingExportersSniffEnabled, + NodeDeprecationChecks::checkMonitoringSettingExportersHeaders, + NodeDeprecationChecks::checkMonitoringSettingExportersTemplateTimeout, + NodeDeprecationChecks::checkMonitoringSettingExportersMasterTimeout, + NodeDeprecationChecks::checkMonitoringSettingExportersEnabled, + NodeDeprecationChecks::checkMonitoringSettingExportersType, + NodeDeprecationChecks::checkMonitoringSettingExportersAlertsEnabled, + NodeDeprecationChecks::checkMonitoringSettingExportersAlertsBlacklist, + NodeDeprecationChecks::checkMonitoringSettingExportersIndexNameTimeFormat, + NodeDeprecationChecks::checkMonitoringSettingDecommissionAlerts, + NodeDeprecationChecks::checkMonitoringSettingEsCollectionEnabled, + NodeDeprecationChecks::checkMonitoringSettingCollectionEnabled, + NodeDeprecationChecks::checkMonitoringSettingCollectionInterval, + NodeDeprecationChecks::checkScriptContextCache, + NodeDeprecationChecks::checkScriptContextCompilationsRateLimitSetting, + NodeDeprecationChecks::checkScriptContextCacheSizeSetting, + NodeDeprecationChecks::checkScriptContextCacheExpirationSetting, + NodeDeprecationChecks::checkEnforceDefaultTierPreferenceSetting, + NodeDeprecationChecks::checkLifecyleStepMasterTimeoutSetting, + NodeDeprecationChecks::checkEqlEnabledSetting, + NodeDeprecationChecks::checkNodeAttrData, + NodeDeprecationChecks::checkWatcherBulkConcurrentRequestsSetting, + NodeDeprecationChecks::checkTracingApmSettings + ); + static DeprecationIssue checkDeprecatedSetting( final Settings clusterSettings, final Settings nodeSettings, @@ -77,15 +131,6 @@ private static Map createMetaMapForRemovableSettings(boolean can return canAutoRemoveSetting ? DeprecationIssue.createMetaMapForRemovableSettings(removableSettings) : null; } - static DeprecationIssue checkRemovedSetting( - final Settings clusterSettings, - final Settings nodeSettings, - final Setting removedSetting, - final String url - ) { - return checkRemovedSetting(clusterSettings, nodeSettings, removedSetting, url, null, DeprecationIssue.Level.CRITICAL); - } - static DeprecationIssue checkRemovedSetting( final Settings clusterSettings, final Settings nodeSettings, @@ -1012,4 +1057,9 @@ static DeprecationIssue checkTracingApmSettings( DeprecationIssue.Level.CRITICAL ); } + + @FunctionalInterface + public interface NodeDeprecationCheck { + R apply(A first, B second, C third, D fourth); + } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ResourceDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ResourceDeprecationChecker.java index 71b9903f69f86..daa3514e3b989 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ResourceDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ResourceDeprecationChecker.java @@ -24,8 +24,14 @@ public interface ResourceDeprecationChecker { * This runs the checks for the current deprecation checker. * * @param clusterState The cluster state provided for the checker + * @param request The deprecation request that triggered this check + * @param precomputedData Data that have been remotely retrieved and might be useful in the checks */ - Map> check(ClusterState clusterState, DeprecationInfoAction.Request request); + Map> check( + ClusterState clusterState, + DeprecationInfoAction.Request request, + TransportDeprecationInfoAction.PrecomputedData precomputedData + ); /** * @return The name of the checker diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationChecker.java index 5a451a9613797..d68e31a1fde4e 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationChecker.java @@ -19,9 +19,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.function.Function; -import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.filterChecks; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_COMMON_DETAIL; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_HELP_URL; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_MESSAGE; @@ -32,20 +32,34 @@ public class TemplateDeprecationChecker implements ResourceDeprecationChecker { public static final String NAME = "templates"; - private static final List> INDEX_TEMPLATE_CHECKS = List.of( - TemplateDeprecationChecker::checkLegacyTiersInIndexTemplate + private final List> indexTemplateChecks = List.of( + this::checkLegacyTiersInIndexTemplate ); - private static final List> COMPONENT_TEMPLATE_CHECKS = List.of( - TemplateDeprecationChecker::checkSourceModeInComponentTemplates, - TemplateDeprecationChecker::checkLegacyTiersInComponentTemplates + private final List> componentTemplateChecks = List.of( + this::checkSourceModeInComponentTemplates, + this::checkLegacyTiersInComponentTemplates ); /** * @param clusterState The cluster state provided for the checker + * @param request not used yet in these checks + * @param precomputedData not used yet in these checks * @return the name of the data streams that have violated the checks with their respective warnings. */ @Override - public Map> check(ClusterState clusterState, DeprecationInfoAction.Request request) { + public Map> check( + ClusterState clusterState, + DeprecationInfoAction.Request request, + TransportDeprecationInfoAction.PrecomputedData precomputedData + ) { + return check(clusterState); + } + + /** + * @param clusterState The cluster state provided for the checker + * @return the name of the data streams that have violated the checks with their respective warnings. + */ + Map> check(ClusterState clusterState) { var indexTemplates = clusterState.metadata().templatesV2().entrySet(); var componentTemplates = clusterState.metadata().componentTemplates().entrySet(); if (indexTemplates.isEmpty() && componentTemplates.isEmpty()) { @@ -56,7 +70,10 @@ public Map> check(ClusterState clusterState, Depr String name = entry.getKey(); ComposableIndexTemplate template = entry.getValue(); - List issuesForSingleIndexTemplate = filterChecks(INDEX_TEMPLATE_CHECKS, c -> c.apply(template)); + List issuesForSingleIndexTemplate = indexTemplateChecks.stream() + .map(c -> c.apply(template)) + .filter(Objects::nonNull) + .toList(); if (issuesForSingleIndexTemplate.isEmpty() == false) { issues.computeIfAbsent(name, ignored -> new ArrayList<>()).addAll(issuesForSingleIndexTemplate); } @@ -65,7 +82,10 @@ public Map> check(ClusterState clusterState, Depr String name = entry.getKey(); ComponentTemplate template = entry.getValue(); - List issuesForSingleIndexTemplate = filterChecks(COMPONENT_TEMPLATE_CHECKS, c -> c.apply(template)); + List issuesForSingleIndexTemplate = componentTemplateChecks.stream() + .map(c -> c.apply(template)) + .filter(Objects::nonNull) + .toList(); if (issuesForSingleIndexTemplate.isEmpty() == false) { issues.computeIfAbsent(name, ignored -> new ArrayList<>()).addAll(issuesForSingleIndexTemplate); } @@ -73,7 +93,7 @@ public Map> check(ClusterState clusterState, Depr return issues.isEmpty() ? Map.of() : issues; } - static DeprecationIssue checkLegacyTiersInIndexTemplate(ComposableIndexTemplate composableIndexTemplate) { + private DeprecationIssue checkLegacyTiersInIndexTemplate(ComposableIndexTemplate composableIndexTemplate) { Template template = composableIndexTemplate.template(); if (template != null) { List deprecatedSettings = LegacyTiersDetection.getDeprecatedFilteredAllocationSettings(template.settings()); @@ -93,7 +113,7 @@ static DeprecationIssue checkLegacyTiersInIndexTemplate(ComposableIndexTemplate return null; } - static DeprecationIssue checkSourceModeInComponentTemplates(ComponentTemplate template) { + private DeprecationIssue checkSourceModeInComponentTemplates(ComponentTemplate template) { if (template.template().mappings() != null) { var sourceAsMap = (Map) XContentHelper.convertToMap(template.template().mappings().uncompressed(), true).v2().get("_doc"); if (sourceAsMap != null) { @@ -115,7 +135,7 @@ static DeprecationIssue checkSourceModeInComponentTemplates(ComponentTemplate te return null; } - static DeprecationIssue checkLegacyTiersInComponentTemplates(ComponentTemplate componentTemplate) { + private DeprecationIssue checkLegacyTiersInComponentTemplates(ComponentTemplate componentTemplate) { Template template = componentTemplate.template(); List deprecatedSettings = LegacyTiersDetection.getDeprecatedFilteredAllocationSettings(template.settings()); if (deprecatedSettings.isEmpty()) { diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransformDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransformDeprecationChecker.java deleted file mode 100644 index 57c4fae960854..0000000000000 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransformDeprecationChecker.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.deprecation; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; -import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; - -import java.util.ArrayList; -import java.util.List; - -class TransformDeprecationChecker implements DeprecationChecker { - - public static final String TRANSFORM_DEPRECATION_KEY = "transform_settings"; - private final List transformConfigs; - - TransformDeprecationChecker(List transformConfigs) { - this.transformConfigs = transformConfigs; - } - - @Override - public boolean enabled(Settings settings) { - // always enabled - return true; - } - - @Override - public void check(Components components, ActionListener deprecationIssueListener) { - ActionListener.completeWith(deprecationIssueListener, () -> { - List allIssues = new ArrayList<>(); - for (var config : transformConfigs) { - allIssues.addAll(config.checkForDeprecations(components.xContentRegistry())); - } - return new CheckResult(getName(), allIssues); - }); - } - - @Override - public String getName() { - return TRANSFORM_DEPRECATION_KEY; - } -} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java index 886eddf82149e..c30d8829c23f3 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java @@ -6,12 +6,11 @@ */ package org.elasticsearch.xpack.deprecation; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.GroupedActionListener; +import org.elasticsearch.action.support.RefCountingListener; import org.elasticsearch.action.support.ThreadedActionListener; import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; import org.elasticsearch.client.internal.OriginSettingClient; @@ -19,14 +18,22 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Tuple; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.transport.Transports; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.action.util.PageParams; @@ -35,24 +42,30 @@ import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.elasticsearch.xpack.deprecation.DeprecationChecks.CLUSTER_SETTINGS_CHECKS; - public class TransportDeprecationInfoAction extends TransportMasterNodeReadAction< DeprecationInfoAction.Request, DeprecationInfoAction.Response> { - private static final DeprecationChecker ML_CHECKER = new MlDeprecationChecker(); - private static final Logger logger = LogManager.getLogger(TransportDeprecationInfoAction.class); + public static final Setting> SKIP_DEPRECATIONS_SETTING = Setting.stringListSetting( + "deprecation.skip_deprecated_settings", + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + private static final List PLUGIN_CHECKERS = List.of(new MlDeprecationChecker()); private final NodeClient client; private final IndexNameExpressionResolver indexNameExpressionResolver; private final Settings settings; private final NamedXContentRegistry xContentRegistry; private volatile List skipTheseDeprecations; + private final NodeDeprecationChecker nodeDeprecationChecker; + private final ClusterDeprecationChecker clusterDeprecationChecker; + private final List resourceDeprecationCheckers; @Inject public TransportDeprecationInfoAction( @@ -79,10 +92,17 @@ public TransportDeprecationInfoAction( this.indexNameExpressionResolver = indexNameExpressionResolver; this.settings = settings; this.xContentRegistry = xContentRegistry; - skipTheseDeprecations = DeprecationChecks.SKIP_DEPRECATIONS_SETTING.get(settings); + skipTheseDeprecations = SKIP_DEPRECATIONS_SETTING.get(settings); + nodeDeprecationChecker = new NodeDeprecationChecker(threadPool); + clusterDeprecationChecker = new ClusterDeprecationChecker(xContentRegistry); + resourceDeprecationCheckers = List.of( + new IndexDeprecationChecker(indexNameExpressionResolver), + new DataStreamDeprecationChecker(indexNameExpressionResolver), + new TemplateDeprecationChecker(), + new IlmPolicyDeprecationChecker() + ); // Safe to register this here because it happens synchronously before the cluster service is started: - clusterService.getClusterSettings() - .addSettingsUpdateConsumer(DeprecationChecks.SKIP_DEPRECATIONS_SETTING, this::setSkipDeprecations); + clusterService.getClusterSettings().addSettingsUpdateConsumer(SKIP_DEPRECATIONS_SETTING, this::setSkipDeprecations); } private void setSkipDeprecations(List skipDeprecations) { @@ -102,56 +122,222 @@ protected final void masterOperation( ClusterState state, final ActionListener listener ) { - NodesDeprecationCheckRequest nodeDepReq = new NodesDeprecationCheckRequest("_all"); - ClientHelper.executeAsyncWithOrigin( - client, - ClientHelper.DEPRECATION_ORIGIN, - NodesDeprecationCheckAction.INSTANCE, - nodeDepReq, - listener.delegateFailureAndWrap((l, response) -> { - if (response.hasFailures()) { - List failedNodeIds = response.failures() - .stream() - .map(failure -> failure.nodeId() + ": " + failure.getMessage()) - .collect(Collectors.toList()); - logger.warn("nodes failed to run deprecation checks: {}", failedNodeIds); - for (FailedNodeException failure : response.failures()) { - logger.debug("node {} failed to run deprecation checks: {}", failure.nodeId(), failure); - } - } - transformConfigs(l.delegateFailureAndWrap((ll, transformConfigs) -> { - DeprecationChecker.Components components = new DeprecationChecker.Components( - xContentRegistry, - settings, - new OriginSettingClient(client, ClientHelper.DEPRECATION_ORIGIN) - ); - pluginSettingIssues( - List.of(ML_CHECKER, new TransformDeprecationChecker(transformConfigs)), - components, - new ThreadedActionListener<>( - client.threadPool().generic(), - ll.map( - deprecationIssues -> DeprecationInfoAction.Response.from( - state, - indexNameExpressionResolver, - request, - response, - CLUSTER_SETTINGS_CHECKS, - deprecationIssues, - skipTheseDeprecations, - List.of( - new IndexDeprecationChecker(indexNameExpressionResolver, indexToTransformIds(transformConfigs)), - new DataStreamDeprecationChecker(indexNameExpressionResolver), - new TemplateDeprecationChecker(), - new IlmPolicyDeprecationChecker() - ) - ) - ) - ) - ); - })); - }) + PrecomputedData precomputedData = new PrecomputedData(); + try (var refs = new RefCountingListener(checkAndCreateResponse(state, request, precomputedData, listener))) { + nodeDeprecationChecker.check(client, refs.acquire(precomputedData::setOnceNodeSettingsIssues)); + transformConfigs(refs.acquire(precomputedData::setOnceTransformConfigs)); + DeprecationChecker.Components components = new DeprecationChecker.Components( + xContentRegistry, + settings, + new OriginSettingClient(client, ClientHelper.DEPRECATION_ORIGIN) + ); + pluginSettingIssues(PLUGIN_CHECKERS, components, refs.acquire(precomputedData::setOncePluginIssues)); + } + } + + /** + * This is the function that does the bulk of the logic of combining the necessary dependencies together, including the cluster state, + * the precalculated information in {@code context} with the remaining checkers such as the cluster setting checker and the resource + * checkers.This function will run a significant part of the checks and build out the final list of issues that exist in the + * cluster. Because of that, it's important that it does not run in the transport thread that's why it's combined with + * {@link #executeInGenericThreadpool(ActionListener)}. + * + * @param state The cluster state + * @param request The originating request containing the index expressions to evaluate + * @param precomputedData Data from remote requests necessary to construct the response + * @param responseListener The listener expecting the {@link DeprecationInfoAction.Response} + * @return The listener that should be executed after all the remote requests have completed and the {@link PrecomputedData} + * is initialised. + */ + public ActionListener checkAndCreateResponse( + ClusterState state, + DeprecationInfoAction.Request request, + PrecomputedData precomputedData, + ActionListener responseListener + ) { + return executeInGenericThreadpool( + ActionListener.running( + () -> responseListener.onResponse( + checkAndCreateResponse( + state, + indexNameExpressionResolver, + request, + skipTheseDeprecations, + clusterDeprecationChecker, + resourceDeprecationCheckers, + precomputedData + ) + ) + ) + ); + } + + /** + * This is the function that does the bulk of the logic of combining the necessary dependencies together, including the cluster state, + * the precalculated information in {@code context} with the remaining checkers such as the cluster setting checker and the resource + * checkers.This function will run a significant part of the checks and build out the final list of issues that exist in the + * cluster. It's important that it does not run in the transport thread that's why it's combined with + * {@link #checkAndCreateResponse(ClusterState, DeprecationInfoAction.Request, PrecomputedData, ActionListener)}. We keep this separated + * for testing purposes. + * + * @param state The cluster state + * @param indexNameExpressionResolver Used to resolve indices into their concrete names + * @param request The originating request containing the index expressions to evaluate + * @param skipTheseDeprecatedSettings the settings that will be removed from cluster metadata and the index metadata of all the + * indexes specified by indexNames + * @param clusterDeprecationChecker The checker that provides the cluster settings deprecations warnings + * @param resourceDeprecationCheckers these are checkers that take as input the cluster state and return a map from resource type + * to issues grouped by the resource name. + * @param precomputedData data from remote requests necessary to construct the response + * @return The list of deprecation issues found in the cluster + */ + static DeprecationInfoAction.Response checkAndCreateResponse( + ClusterState state, + IndexNameExpressionResolver indexNameExpressionResolver, + DeprecationInfoAction.Request request, + List skipTheseDeprecatedSettings, + ClusterDeprecationChecker clusterDeprecationChecker, + List resourceDeprecationCheckers, + PrecomputedData precomputedData + ) { + assert Transports.assertNotTransportThread("walking mappings in indexSettingsChecks is expensive"); + // Allow system index access here to prevent deprecation warnings when we call this API + String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames(state, request); + ClusterState stateWithSkippedSettingsRemoved = removeSkippedSettings(state, concreteIndexNames, skipTheseDeprecatedSettings); + List clusterSettingsIssues = clusterDeprecationChecker.check( + stateWithSkippedSettingsRemoved, + precomputedData.transformConfigs() + ); + + Map>> resourceDeprecationIssues = new HashMap<>(); + for (ResourceDeprecationChecker resourceDeprecationChecker : resourceDeprecationCheckers) { + Map> issues = resourceDeprecationChecker.check( + stateWithSkippedSettingsRemoved, + request, + precomputedData + ); + if (issues.isEmpty() == false) { + resourceDeprecationIssues.put(resourceDeprecationChecker.getName(), issues); + } + } + + return new DeprecationInfoAction.Response( + clusterSettingsIssues, + precomputedData.nodeSettingsIssues(), + resourceDeprecationIssues, + precomputedData.pluginIssues() + ); + } + + /** + * This class holds the results of remote requests. These can be either checks that require remote requests such as + * {@code nodeSettingsIssues} and {@code pluginIssues} or metadata needed for more than one types of checks such as + * {@code transformConfigs}. + */ + public static class PrecomputedData { + private final SetOnce> nodeSettingsIssues = new SetOnce<>(); + private final SetOnce>> pluginIssues = new SetOnce<>(); + private final SetOnce> transformConfigs = new SetOnce<>(); + + public void setOnceNodeSettingsIssues(List nodeSettingsIssues) { + this.nodeSettingsIssues.set(nodeSettingsIssues); + } + + public void setOncePluginIssues(Map> pluginIssues) { + this.pluginIssues.set(pluginIssues); + } + + public void setOnceTransformConfigs(List transformConfigs) { + this.transformConfigs.set(transformConfigs); + } + + public List nodeSettingsIssues() { + return nodeSettingsIssues.get(); + } + + public Map> pluginIssues() { + return pluginIssues.get(); + } + + public List transformConfigs() { + return transformConfigs.get(); + } + } + + /** + * Removes the skipped settings from the selected indices and the component and index templates. + * @param state The cluster state to modify + * @param indexNames The names of the indexes whose settings need to be filtered + * @param skipTheseDeprecatedSettings The settings that will be removed from cluster metadata and the index metadata of all the + * indexes specified by indexNames + * @return A modified cluster state with the given settings removed + */ + private static ClusterState removeSkippedSettings(ClusterState state, String[] indexNames, List skipTheseDeprecatedSettings) { + // Short-circuit, no need to reconstruct the cluster state if there are no settings to remove + if (skipTheseDeprecatedSettings == null || skipTheseDeprecatedSettings.isEmpty()) { + return state; + } + ClusterState.Builder clusterStateBuilder = new ClusterState.Builder(state); + Metadata.Builder metadataBuilder = Metadata.builder(state.metadata()); + metadataBuilder.transientSettings( + metadataBuilder.transientSettings().filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false) + ); + metadataBuilder.persistentSettings( + metadataBuilder.persistentSettings().filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false) ); + Map indicesBuilder = new HashMap<>(state.getMetadata().indices()); + for (String indexName : indexNames) { + IndexMetadata indexMetadata = state.getMetadata().index(indexName); + IndexMetadata.Builder filteredIndexMetadataBuilder = new IndexMetadata.Builder(indexMetadata); + Settings filteredSettings = indexMetadata.getSettings() + .filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false); + filteredIndexMetadataBuilder.settings(filteredSettings); + indicesBuilder.put(indexName, filteredIndexMetadataBuilder.build()); + } + metadataBuilder.componentTemplates(state.metadata().componentTemplates().entrySet().stream().map(entry -> { + String templateName = entry.getKey(); + ComponentTemplate componentTemplate = entry.getValue(); + Template template = componentTemplate.template(); + if (template.settings() == null || template.settings().isEmpty()) { + return Tuple.tuple(templateName, componentTemplate); + } + return Tuple.tuple( + templateName, + new ComponentTemplate( + Template.builder(template) + .settings(template.settings().filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false)) + .build(), + componentTemplate.version(), + componentTemplate.metadata(), + componentTemplate.deprecated() + ) + ); + }).collect(Collectors.toMap(Tuple::v1, Tuple::v2))); + metadataBuilder.indexTemplates(state.metadata().templatesV2().entrySet().stream().map(entry -> { + String templateName = entry.getKey(); + ComposableIndexTemplate indexTemplate = entry.getValue(); + Template template = indexTemplate.template(); + if (template == null || template.settings() == null || template.settings().isEmpty()) { + return Tuple.tuple(templateName, indexTemplate); + } + return Tuple.tuple( + templateName, + indexTemplate.toBuilder() + .template( + Template.builder(indexTemplate.template()) + .settings( + indexTemplate.template() + .settings() + .filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false) + ) + ) + .build() + ); + }).collect(Collectors.toMap(Tuple::v1, Tuple::v2))); + + metadataBuilder.indices(indicesBuilder); + clusterStateBuilder.metadata(metadataBuilder); + return clusterStateBuilder.build(); } static void pluginSettingIssues( @@ -192,34 +378,21 @@ private void transformConfigs(PageParams currentPage, ActionListener( - threadPool.generic(), - currentPageListener.delegateFailureAndWrap((delegate, getTransformConfigResponse) -> { - var currentPageOfConfigs = getTransformConfigResponse.getTransformConfigurations().stream(); - var currentPageSize = currentPage.getFrom() + currentPage.getSize(); - var totalTransformConfigCount = getTransformConfigResponse.getTransformConfigurationCount(); - if (totalTransformConfigCount >= currentPageSize) { - var nextPage = new PageParams(currentPageSize, PageParams.DEFAULT_SIZE); - transformConfigs( - nextPage, - delegate.map(nextPageOfConfigs -> Stream.concat(currentPageOfConfigs, nextPageOfConfigs)) - ); - } else { - delegate.onResponse(currentPageOfConfigs); - } - }) - ) + executeInGenericThreadpool(currentPageListener.delegateFailureAndWrap((delegate, getTransformConfigResponse) -> { + var currentPageOfConfigs = getTransformConfigResponse.getTransformConfigurations().stream(); + var currentPageSize = currentPage.getFrom() + currentPage.getSize(); + var totalTransformConfigCount = getTransformConfigResponse.getTransformConfigurationCount(); + if (totalTransformConfigCount >= currentPageSize) { + var nextPage = new PageParams(currentPageSize, PageParams.DEFAULT_SIZE); + transformConfigs(nextPage, delegate.map(nextPageOfConfigs -> Stream.concat(currentPageOfConfigs, nextPageOfConfigs))); + } else { + delegate.onResponse(currentPageOfConfigs); + } + })) ); } - private Map> indexToTransformIds(List transformConfigs) { - return transformConfigs.stream() - .collect( - Collectors.groupingBy( - config -> config.getDestination().getIndex(), - Collectors.mapping(TransformConfig::getId, Collectors.toList()) - ) - ); + private ActionListener executeInGenericThreadpool(ActionListener listener) { + return new ThreadedActionListener<>(threadPool.generic(), listener); } - } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java index 745f5e7ae8959..befe0bd6b41a4 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java @@ -36,6 +36,7 @@ import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.Objects; import static org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING; @@ -75,10 +76,10 @@ public TransportNodeDeprecationCheckAction( this.pluginsService = pluginsService; this.licenseState = licenseState; this.clusterInfoService = clusterInfoService; - skipTheseDeprecations = DeprecationChecks.SKIP_DEPRECATIONS_SETTING.get(settings); + skipTheseDeprecations = TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING.get(settings); // Safe to register this here because it happens synchronously before the cluster service is started: clusterService.getClusterSettings() - .addSettingsUpdateConsumer(DeprecationChecks.SKIP_DEPRECATIONS_SETTING, this::setSkipDeprecations); + .addSettingsUpdateConsumer(TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING, this::setSkipDeprecations); } private void setSkipDeprecations(List skipDeprecations) { @@ -106,13 +107,13 @@ protected NodesDeprecationCheckAction.NodeResponse newNodeResponse(StreamInput i @Override protected NodesDeprecationCheckAction.NodeResponse nodeOperation(NodesDeprecationCheckAction.NodeRequest request, Task task) { - return nodeOperation(request, DeprecationChecks.NODE_SETTINGS_CHECKS); + return nodeOperation(request, NodeDeprecationChecks.SINGLE_NODE_CHECKS); } NodesDeprecationCheckAction.NodeResponse nodeOperation( NodesDeprecationCheckAction.NodeRequest request, List< - DeprecationChecks.NodeDeprecationCheck< + NodeDeprecationChecks.NodeDeprecationCheck< Settings, PluginsAndModules, ClusterState, @@ -130,10 +131,10 @@ NodesDeprecationCheckAction.NodeResponse nodeOperation( .metadata(Metadata.builder(metadata).transientSettings(transientSettings).persistentSettings(persistentSettings).build()) .build(); - List issues = DeprecationInfoAction.filterChecks( - nodeSettingsChecks, - (c) -> c.apply(filteredNodeSettings, pluginsService.info(), filteredClusterState, licenseState) - ); + List issues = nodeSettingsChecks.stream() + .map(c -> c.apply(filteredNodeSettings, pluginsService.info(), filteredClusterState, licenseState)) + .filter(Objects::nonNull) + .toList(); DeprecationIssue watermarkIssue = checkDiskLowWatermark( filteredNodeSettings, filteredClusterState.metadata().settings(), diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java index 450dee7a76a98..a8dd1d464e30c 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java @@ -69,7 +69,7 @@ public void testOldIndicesCheck() { ); // We know that the data stream checks ignore the request. - Map> issuesByDataStream = checker.check(clusterState, null); + Map> issuesByDataStream = checker.check(clusterState); assertThat(issuesByDataStream.size(), equalTo(1)); assertThat(issuesByDataStream.containsKey(dataStream.getName()), equalTo(true)); assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); @@ -91,7 +91,7 @@ public void testOldIndicesCheckWithOnlyNewIndices() { .build(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); - Map> issuesByDataStream = checker.check(clusterState, null); + Map> issuesByDataStream = checker.check(clusterState); assertThat(issuesByDataStream.size(), equalTo(0)); } @@ -137,7 +137,7 @@ public void testOldIndicesCheckWithClosedAndOpenIndices() { ) ); - Map> issuesByDataStream = checker.check(clusterState, null); + Map> issuesByDataStream = checker.check(clusterState); assertThat(issuesByDataStream.containsKey(dataStream.getName()), equalTo(true)); assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); } @@ -297,7 +297,7 @@ public void testOldIndicesIgnoredWarningCheck() { ) ); - Map> issuesByDataStream = checker.check(clusterState, null); + Map> issuesByDataStream = checker.check(clusterState); assertThat(issuesByDataStream.containsKey(dataStream.getName()), equalTo(true)); assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationChecksTests.java deleted file mode 100644 index 9a57450b7fad7..0000000000000 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationChecksTests.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.deprecation; - -import org.elasticsearch.core.Tuple; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; - -import java.util.ArrayList; -import java.util.List; -import java.util.function.Supplier; - -import static org.hamcrest.Matchers.equalTo; - -public class DeprecationChecksTests extends ESTestCase { - - public void testFilterChecks() { - DeprecationIssue issue = createRandomDeprecationIssue(); - int numChecksPassed = randomIntBetween(0, 5); - int numChecksFailed = 10 - numChecksPassed; - List> checks = new ArrayList<>(); - for (int i = 0; i < numChecksFailed; i++) { - checks.add(() -> issue); - } - for (int i = 0; i < numChecksPassed; i++) { - checks.add(() -> null); - } - List filteredIssues = DeprecationInfoAction.filterChecks(checks, Supplier::get); - assertThat(filteredIssues.size(), equalTo(numChecksFailed)); - } - - private static DeprecationIssue createRandomDeprecationIssue() { - String details = randomBoolean() ? randomAlphaOfLength(10) : null; - return new DeprecationIssue( - randomFrom(DeprecationIssue.Level.values()), - randomAlphaOfLength(10), - randomAlphaOfLength(10), - details, - randomBoolean(), - randomMap(1, 5, () -> Tuple.tuple(randomAlphaOfLength(4), randomAlphaOfLength(4))) - ); - } -} diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java index a06bb2f2718cb..537c3eb84a902 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java @@ -6,50 +6,17 @@ */ package org.elasticsearch.xpack.deprecation; -import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.ComponentTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.metadata.Template; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNodeUtils; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.core.Tuple; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue.Level; -import org.junit.Assert; -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.BiFunction; -import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.Response.RESERVED_NAMES; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.core.IsEqual.equalTo; - public class DeprecationInfoActionResponseTests extends AbstractWireSerializingTestCase { @Override @@ -153,329 +120,11 @@ protected Writeable.Reader instanceReader() { return DeprecationInfoAction.Response::new; } - public void testFrom() throws IOException { - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all"); - mapping.field("enabled", false); - mapping.endObject().endObject(); - - Metadata metadata = Metadata.builder() - .put( - IndexMetadata.builder("test") - .putMapping(Strings.toString(mapping)) - .settings(settings(IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - ) - .build(); - - DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("test", new TransportAddress(TransportAddress.META_ADDRESS, 9300)); - ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); - IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); - boolean clusterIssueFound = randomBoolean(); - boolean nodeIssueFound = randomBoolean(); - boolean indexIssueFound = randomBoolean(); - boolean dataStreamIssueFound = randomBoolean(); - boolean indexTemplateIssueFound = randomBoolean(); - boolean componentTemplateIssueFound = randomBoolean(); - boolean ilmPolicyIssueFound = randomBoolean(); - DeprecationIssue foundIssue = createTestDeprecationIssue(); - List> clusterSettingsChecks = List.of((s) -> clusterIssueFound ? foundIssue : null); - List resourceCheckers = List.of(createResourceChecker("index_settings", (cs, req) -> { - if (indexIssueFound) { - return Map.of("test", List.of(foundIssue)); - } - return Map.of(); - }), createResourceChecker("data_streams", (cs, req) -> { - if (dataStreamIssueFound) { - return Map.of("my-ds", List.of(foundIssue)); - } - return Map.of(); - }), createResourceChecker("templates", (cs, req) -> { - Map> issues = new HashMap<>(); - if (componentTemplateIssueFound) { - issues.put("my-component-template", List.of(foundIssue)); - } - if (indexTemplateIssueFound) { - issues.put("my-index-template", List.of(foundIssue)); - } - return issues; - }), createResourceChecker("ilm_policies", (cs, req) -> { - if (ilmPolicyIssueFound) { - return Map.of("my-policy", List.of(foundIssue)); - } - return Map.of(); - })); - - NodesDeprecationCheckResponse nodeDeprecationIssues = new NodesDeprecationCheckResponse( - new ClusterName(randomAlphaOfLength(5)), - nodeIssueFound ? List.of(new NodesDeprecationCheckAction.NodeResponse(discoveryNode, List.of(foundIssue))) : List.of(), - List.of() - ); - - DeprecationInfoAction.Request request = new DeprecationInfoAction.Request(randomTimeValue(), Strings.EMPTY_ARRAY); - DeprecationInfoAction.Response response = DeprecationInfoAction.Response.from( - state, - resolver, - request, - nodeDeprecationIssues, - clusterSettingsChecks, - new HashMap<>(), // modified in the method to move transform deprecation issues into cluster_settings - List.of(), - resourceCheckers - ); - - if (clusterIssueFound) { - assertThat(response.getClusterSettingsIssues(), equalTo(List.of(foundIssue))); - } else { - assertThat(response.getClusterSettingsIssues(), empty()); - } - - if (nodeIssueFound) { - String details = foundIssue.getDetails() != null ? foundIssue.getDetails() + " " : ""; - DeprecationIssue mergedFoundIssue = new DeprecationIssue( - foundIssue.getLevel(), - foundIssue.getMessage(), - foundIssue.getUrl(), - details + "(nodes impacted: [" + discoveryNode.getName() + "])", - foundIssue.isResolveDuringRollingUpgrade(), - foundIssue.getMeta() - ); - assertThat(response.getNodeSettingsIssues(), equalTo(List.of(mergedFoundIssue))); - } else { - assertTrue(response.getNodeSettingsIssues().isEmpty()); - } - - if (indexIssueFound) { - assertThat(response.getIndexSettingsIssues(), equalTo(Map.of("test", List.of(foundIssue)))); - } else { - assertTrue(response.getIndexSettingsIssues().isEmpty()); - } - if (dataStreamIssueFound) { - assertThat(response.getDataStreamDeprecationIssues(), equalTo(Map.of("my-ds", List.of(foundIssue)))); - } else { - assertTrue(response.getDataStreamDeprecationIssues().isEmpty()); - } - if (ilmPolicyIssueFound) { - assertThat(response.getIlmPolicyDeprecationIssues(), equalTo(Map.of("my-policy", List.of(foundIssue)))); - } else { - assertTrue(response.getIlmPolicyDeprecationIssues().isEmpty()); - } - if (componentTemplateIssueFound == false && indexTemplateIssueFound == false) { - assertTrue(response.getTemplateDeprecationIssues().isEmpty()); - } else { - if (componentTemplateIssueFound) { - assertThat(response.getTemplateDeprecationIssues().get("my-component-template"), equalTo(List.of(foundIssue))); - } - if (indexTemplateIssueFound) { - assertThat(response.getTemplateDeprecationIssues().get("my-index-template"), equalTo(List.of(foundIssue))); - } - - } - } - - public void testFromWithMergeableNodeIssues() throws IOException { - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all"); - mapping.field("enabled", false); - mapping.endObject().endObject(); - - Metadata metadata = Metadata.builder() - .put( - IndexMetadata.builder("test") - .putMapping(Strings.toString(mapping)) - .settings(settings(IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - ) - .build(); - - DiscoveryNode node1 = DiscoveryNodeUtils.builder("nodeId1") - .name("node1") - .ephemeralId("ephemeralId1") - .address("hostName1", "hostAddress1", new TransportAddress(TransportAddress.META_ADDRESS, 9300)) - .roles(Set.of()) - .build(); - DiscoveryNode node2 = DiscoveryNodeUtils.builder("nodeId2") - .name("node2") - .ephemeralId("ephemeralId2") - .address("hostName2", "hostAddress2", new TransportAddress(TransportAddress.META_ADDRESS, 9500)) - .roles(Set.of()) - .build(); - ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); - IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); - Map metaMap1 = DeprecationIssue.createMetaMapForRemovableSettings(List.of("setting.1", "setting.2", "setting.3")); - Map metaMap2 = DeprecationIssue.createMetaMapForRemovableSettings(List.of("setting.2", "setting.3")); - DeprecationIssue foundIssue1 = createTestDeprecationIssue(metaMap1); - DeprecationIssue foundIssue2 = createTestDeprecationIssue(foundIssue1, metaMap2); - List> clusterSettingsChecks = List.of(); - List resourceCheckers = List.of(); - - NodesDeprecationCheckResponse nodeDeprecationIssues = new NodesDeprecationCheckResponse( - new ClusterName(randomAlphaOfLength(5)), - Arrays.asList( - new NodesDeprecationCheckAction.NodeResponse(node1, List.of(foundIssue1)), - new NodesDeprecationCheckAction.NodeResponse(node2, List.of(foundIssue2)) - ), - List.of() - ); - - DeprecationInfoAction.Request request = new DeprecationInfoAction.Request(randomTimeValue(), Strings.EMPTY_ARRAY); - DeprecationInfoAction.Response response = DeprecationInfoAction.Response.from( - state, - resolver, - request, - nodeDeprecationIssues, - clusterSettingsChecks, - new HashMap<>(), // modified in the method to move transform deprecation issues into cluster_settings - List.of(), - resourceCheckers - ); - - String details = foundIssue1.getDetails() != null ? foundIssue1.getDetails() + " " : ""; - DeprecationIssue mergedFoundIssue = new DeprecationIssue( - foundIssue1.getLevel(), - foundIssue1.getMessage(), - foundIssue1.getUrl(), - details + "(nodes impacted: [" + node1.getName() + ", " + node2.getName() + "])", - foundIssue1.isResolveDuringRollingUpgrade(), - foundIssue2.getMeta() - ); - assertThat(response.getNodeSettingsIssues(), equalTo(List.of(mergedFoundIssue))); - } - - public void testRemoveSkippedSettings() { - Settings.Builder settingsBuilder = settings(IndexVersion.current()); - settingsBuilder.put("some.deprecated.property", "someValue1"); - settingsBuilder.put("some.other.bad.deprecated.property", "someValue2"); - settingsBuilder.put("some.undeprecated.property", "someValue3"); - settingsBuilder.putList("some.undeprecated.list.property", List.of("someValue4", "someValue5")); - Settings inputSettings = settingsBuilder.build(); - IndexMetadata dataStreamIndexMetadata = IndexMetadata.builder("ds-test-index-1") - .settings(inputSettings) - .numberOfShards(1) - .numberOfReplicas(0) - .build(); - ComponentTemplate componentTemplate = new ComponentTemplate(Template.builder().settings(inputSettings).build(), null, null); - ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() - .template(Template.builder().settings(inputSettings)) - .build(); - Metadata metadata = Metadata.builder() - .put(IndexMetadata.builder("test").settings(inputSettings).numberOfShards(1).numberOfReplicas(0)) - .put(dataStreamIndexMetadata, true) - .put(DataStream.builder("ds-test", List.of(dataStreamIndexMetadata.getIndex())).build()) - .indexTemplates( - Map.of( - "my-index-template", - indexTemplate, - "empty-template", - ComposableIndexTemplate.builder().indexPatterns(List.of("random")).build() - ) - ) - .componentTemplates(Map.of("my-component-template", componentTemplate)) - .persistentSettings(inputSettings) - .build(); - - ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); - IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); - AtomicReference visibleClusterSettings = new AtomicReference<>(); - List> clusterSettingsChecks = List.of((s) -> { - visibleClusterSettings.set(s.getMetadata().settings()); - return null; - }); - AtomicReference visibleIndexSettings = new AtomicReference<>(); - AtomicReference visibleComponentTemplateSettings = new AtomicReference<>(); - AtomicReference visibleIndexTemplateSettings = new AtomicReference<>(); - AtomicInteger backingIndicesCount = new AtomicInteger(0); - List resourceCheckers = List.of(createResourceChecker("index_settings", (cs, req) -> { - for (String indexName : resolver.concreteIndexNames(cs, req)) { - visibleIndexSettings.set(cs.metadata().index(indexName).getSettings()); - } - return Map.of(); - }), createResourceChecker("data_streams", (cs, req) -> { - cs.metadata().dataStreams().values().forEach(ds -> backingIndicesCount.set(ds.getIndices().size())); - return Map.of(); - }), createResourceChecker("templates", (cs, req) -> { - cs.metadata() - .componentTemplates() - .values() - .forEach(template -> visibleComponentTemplateSettings.set(template.template().settings())); - cs.metadata().templatesV2().values().forEach(template -> { - if (template.template() != null && template.template().settings() != null) { - visibleIndexTemplateSettings.set(template.template().settings()); - } - }); - return Map.of(); - })); - - NodesDeprecationCheckResponse nodeDeprecationIssues = new NodesDeprecationCheckResponse( - new ClusterName(randomAlphaOfLength(5)), - List.of(), - List.of() - ); - - DeprecationInfoAction.Request request = new DeprecationInfoAction.Request(randomTimeValue(), Strings.EMPTY_ARRAY); - DeprecationInfoAction.Response.from( - state, - resolver, - request, - nodeDeprecationIssues, - clusterSettingsChecks, - new HashMap<>(), // modified in the method to move transform deprecation issues into cluster_settings - List.of("some.deprecated.property", "some.other.*.deprecated.property"), - resourceCheckers - ); - - settingsBuilder = settings(IndexVersion.current()); - settingsBuilder.put("some.undeprecated.property", "someValue3"); - settingsBuilder.putList("some.undeprecated.list.property", List.of("someValue4", "someValue5")); - - Settings expectedSettings = settingsBuilder.build(); - Settings resultClusterSettings = visibleClusterSettings.get(); - Assert.assertNotNull(resultClusterSettings); - Assert.assertEquals(expectedSettings, visibleClusterSettings.get()); - - Settings resultIndexSettings = visibleIndexSettings.get(); - Assert.assertNotNull(resultIndexSettings); - Assert.assertEquals("someValue3", resultIndexSettings.get("some.undeprecated.property")); - Assert.assertEquals(resultIndexSettings.getAsList("some.undeprecated.list.property"), List.of("someValue4", "someValue5")); - Assert.assertFalse(resultIndexSettings.hasValue("some.deprecated.property")); - Assert.assertFalse(resultIndexSettings.hasValue("some.other.bad.deprecated.property")); - - assertThat(backingIndicesCount.get(), equalTo(1)); - - Assert.assertNotNull(visibleComponentTemplateSettings.get()); - Assert.assertEquals(expectedSettings, visibleComponentTemplateSettings.get()); - Assert.assertNotNull(visibleIndexTemplateSettings.get()); - Assert.assertEquals(expectedSettings, visibleIndexTemplateSettings.get()); - } - - public void testCtorFailure() { - Map> indexNames = Stream.generate(() -> randomAlphaOfLength(10)) - .limit(10) - .collect(Collectors.toMap(Function.identity(), (_k) -> List.of())); - Map> dataStreamNames = Stream.generate(() -> randomAlphaOfLength(10)) - .limit(10) - .collect(Collectors.toMap(Function.identity(), (_k) -> List.of())); - Set shouldCauseFailure = new HashSet<>(RESERVED_NAMES); - for (int i = 0; i < NUMBER_OF_TEST_RUNS; i++) { - Map> pluginSettingsIssues = randomSubsetOf(3, shouldCauseFailure).stream() - .collect(Collectors.toMap(Function.identity(), (_k) -> List.of())); - expectThrows( - ElasticsearchStatusException.class, - () -> new DeprecationInfoAction.Response( - List.of(), - List.of(), - Map.of("data_streams", dataStreamNames, "index_settings", indexNames), - pluginSettingsIssues - ) - ); - } - } - - private static DeprecationIssue createTestDeprecationIssue() { + static DeprecationIssue createTestDeprecationIssue() { return createTestDeprecationIssue(randomMap(1, 5, () -> Tuple.tuple(randomAlphaOfLength(4), randomAlphaOfLength(4)))); } - private static DeprecationIssue createTestDeprecationIssue(Map metaMap) { + static DeprecationIssue createTestDeprecationIssue(Map metaMap) { String details = randomBoolean() ? randomAlphaOfLength(10) : null; return new DeprecationIssue( randomFrom(Level.values()), @@ -487,7 +136,7 @@ private static DeprecationIssue createTestDeprecationIssue(Map m ); } - private static DeprecationIssue createTestDeprecationIssue(DeprecationIssue seedIssue, Map metaMap) { + static DeprecationIssue createTestDeprecationIssue(DeprecationIssue seedIssue, Map metaMap) { return new DeprecationIssue( seedIssue.getLevel(), seedIssue.getMessage(), @@ -498,27 +147,9 @@ private static DeprecationIssue createTestDeprecationIssue(DeprecationIssue seed ); } - private static List randomDeprecationIssues() { + static List randomDeprecationIssues() { return Stream.generate(DeprecationInfoActionResponseTests::createTestDeprecationIssue) .limit(randomIntBetween(0, 10)) .collect(Collectors.toList()); } - - private static ResourceDeprecationChecker createResourceChecker( - String name, - BiFunction>> check - ) { - return new ResourceDeprecationChecker() { - - @Override - public Map> check(ClusterState clusterState, DeprecationInfoAction.Request request) { - return check.apply(clusterState, request); - } - - @Override - public String getName() { - return name; - } - }; - } } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationCheckerTests.java index 2032a6faedc92..475cd3e6a24bc 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationCheckerTests.java @@ -90,7 +90,7 @@ public void testLegacyTierSettings() { ) .build(); - Map> issuesByComponentTemplate = checker.check(clusterState, null); + Map> issuesByComponentTemplate = checker.check(clusterState); final DeprecationIssue expected = new DeprecationIssue( DeprecationIssue.Level.WARNING, "Configuring tiers via filtered allocation is not recommended.", @@ -136,7 +136,7 @@ public void testFrozenAction() { ) .build(); - Map> issuesByComponentTemplate = checker.check(clusterState, null); + Map> issuesByComponentTemplate = checker.check(clusterState); final DeprecationIssue expected = new DeprecationIssue( DeprecationIssue.Level.WARNING, "ILM policy [deprecated-action] contains the action 'freeze' that is deprecated and will be removed in a future version.", diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java index ea30c93bfb5b8..296ae31cd9f24 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java @@ -29,7 +29,11 @@ import org.elasticsearch.snapshots.SearchableSnapshotsSettings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; +import org.elasticsearch.xpack.core.transform.transforms.DestConfig; +import org.elasticsearch.xpack.core.transform.transforms.SourceConfig; +import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -42,10 +46,18 @@ public class IndexDeprecationCheckerTests extends ESTestCase { + private static final IndexVersion OLD_VERSION = IndexVersion.fromId(7170099); + private final IndexNameExpressionResolver indexNameExpressionResolver = TestIndexNameExpressionResolver.newInstance(); + private final IndexDeprecationChecker checker = new IndexDeprecationChecker(indexNameExpressionResolver); + private final TransportDeprecationInfoAction.PrecomputedData emptyPrecomputedData = + new TransportDeprecationInfoAction.PrecomputedData(); private final IndexMetadata.State indexMetdataState; public IndexDeprecationCheckerTests(@Name("indexMetadataState") IndexMetadata.State indexMetdataState) { this.indexMetdataState = indexMetdataState; + emptyPrecomputedData.setOnceNodeSettingsIssues(List.of()); + emptyPrecomputedData.setOncePluginIssues(Map.of()); + emptyPrecomputedData.setOnceTransformConfigs(List.of()); } @ParametersFactory @@ -53,11 +65,6 @@ public static List createParameters() { return List.of(new Object[] { IndexMetadata.State.OPEN }, new Object[] { IndexMetadata.State.CLOSE }); } - private static final IndexVersion OLD_VERSION = IndexVersion.fromId(7170099); - - private final IndexNameExpressionResolver indexNameExpressionResolver = TestIndexNameExpressionResolver.newInstance(); - private final IndexDeprecationChecker checker = new IndexDeprecationChecker(indexNameExpressionResolver, Map.of()); - public void testOldIndicesCheck() { IndexMetadata indexMetadata = IndexMetadata.builder("test") .settings(settings(OLD_VERSION)) @@ -79,14 +86,15 @@ public void testOldIndicesCheck() { ); Map> issuesByIndex = checker.check( clusterState, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); List issues = issuesByIndex.get("test"); assertEquals(singletonList(expected), issues); } public void testOldTransformIndicesCheck() { - var checker = new IndexDeprecationChecker(indexNameExpressionResolver, Map.of("test", List.of("test-transform"))); + var checker = new IndexDeprecationChecker(indexNameExpressionResolver); var indexMetadata = indexMetadata("test", OLD_VERSION); var clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) .metadata(Metadata.builder().put(indexMetadata, true)) @@ -100,15 +108,15 @@ public void testOldTransformIndicesCheck() { false, Map.of("reindex_required", true, "transform_ids", List.of("test-transform")) ); - var issuesByIndex = checker.check(clusterState, new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS)); + var issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + createContextWithTransformConfigs(Map.of("test", List.of("test-transform"))) + ); assertEquals(singletonList(expected), issuesByIndex.get("test")); } public void testOldIndicesCheckWithMultipleTransforms() { - var checker = new IndexDeprecationChecker( - indexNameExpressionResolver, - Map.of("test", List.of("test-transform1", "test-transform2")) - ); var indexMetadata = indexMetadata("test", OLD_VERSION); var clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) .metadata(Metadata.builder().put(indexMetadata, true)) @@ -122,15 +130,15 @@ public void testOldIndicesCheckWithMultipleTransforms() { false, Map.of("reindex_required", true, "transform_ids", List.of("test-transform1", "test-transform2")) ); - var issuesByIndex = checker.check(clusterState, new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS)); + var issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + createContextWithTransformConfigs(Map.of("test", List.of("test-transform1", "test-transform2"))) + ); assertEquals(singletonList(expected), issuesByIndex.get("test")); } public void testMultipleOldIndicesCheckWithTransforms() { - var checker = new IndexDeprecationChecker( - indexNameExpressionResolver, - Map.of("test1", List.of("test-transform1"), "test2", List.of("test-transform2")) - ); var indexMetadata1 = indexMetadata("test1", OLD_VERSION); var indexMetadata2 = indexMetadata("test2", OLD_VERSION); var clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) @@ -161,7 +169,11 @@ public void testMultipleOldIndicesCheckWithTransforms() { ) ) ); - var issuesByIndex = checker.check(clusterState, new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS)); + var issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + createContextWithTransformConfigs(Map.of("test1", List.of("test-transform1"), "test2", List.of("test-transform2"))) + ); assertEquals(expected, issuesByIndex); } @@ -215,7 +227,8 @@ public void testOldIndicesCheckDataStreamIndex() { .build(); Map> issuesByIndex = checker.check( clusterState, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); assertThat(issuesByIndex.size(), equalTo(0)); } @@ -236,7 +249,8 @@ public void testOldIndicesCheckSnapshotIgnored() { Map> issuesByIndex = checker.check( clusterState, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); assertThat(issuesByIndex.size(), equalTo(0)); } @@ -263,7 +277,8 @@ public void testOldIndicesIgnoredWarningCheck() { ); Map> issuesByIndex = checker.check( clusterState, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); assertTrue(issuesByIndex.containsKey("test")); assertEquals(List.of(expected), issuesByIndex.get("test")); @@ -285,7 +300,8 @@ public void testTranslogRetentionSettings() { .build(); Map> issuesByIndex = checker.check( state, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); List issues = issuesByIndex.get("test"); assertThat( @@ -328,7 +344,8 @@ public void testDefaultTranslogRetentionSettings() { .build(); Map> issuesByIndex = checker.check( state, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); assertThat(issuesByIndex.size(), equalTo(0)); } @@ -348,7 +365,8 @@ public void testIndexDataPathSetting() { .build(); Map> issuesByIndex = checker.check( state, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); final String expectedUrl = "https://www.elastic.co/guide/en/elasticsearch/reference/7.13/breaking-changes-7.13.html#deprecate-shared-data-path-setting"; @@ -382,7 +400,8 @@ public void testSimpleFSSetting() { .build(); Map> issuesByIndex = checker.check( state, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); assertThat( issuesByIndex.get("test"), @@ -425,7 +444,8 @@ public void testCamelCaseDeprecation() { .build(); Map> issuesByIndex = checker.check( state, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); DeprecationIssue expected = new DeprecationIssue( DeprecationIssue.Level.CRITICAL, @@ -456,7 +476,8 @@ public void testLegacyTierIndex() { .build(); Map> issuesByIndex = checker.check( state, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); assertThat( issuesByIndex.get("test"), @@ -484,4 +505,23 @@ private ClusterBlocks clusterBlocksForIndices(IndexMetadata... indicesMetadatas) } return builder.build(); } + + private TransportDeprecationInfoAction.PrecomputedData createContextWithTransformConfigs(Map> indexToTransform) { + List transforms = new ArrayList<>(); + for (Map.Entry> entry : indexToTransform.entrySet()) { + String index = entry.getKey(); + for (String transform : entry.getValue()) { + transforms.add( + TransformConfig.builder() + .setId(transform) + .setSource(new SourceConfig(randomAlphaOfLength(10))) + .setDest(new DestConfig(index, List.of(), null)) + .build() + ); + } + } + TransportDeprecationInfoAction.PrecomputedData precomputedData = new TransportDeprecationInfoAction.PrecomputedData(); + precomputedData.setOnceTransformConfigs(transforms); + return precomputedData; + } } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationCheckerTests.java new file mode 100644 index 0000000000000..78ddba87b9f85 --- /dev/null +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationCheckerTests.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.deprecation; + +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.xpack.deprecation.DeprecationInfoActionResponseTests.createTestDeprecationIssue; +import static org.hamcrest.core.IsEqual.equalTo; + +public class NodeDeprecationCheckerTests extends ESTestCase { + + public void testMergingNodeIssues() throws IOException { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all"); + mapping.field("enabled", false); + mapping.endObject().endObject(); + + DiscoveryNode node1 = DiscoveryNodeUtils.builder("nodeId1") + .name("node1") + .ephemeralId("ephemeralId1") + .address("hostName1", "hostAddress1", new TransportAddress(TransportAddress.META_ADDRESS, 9300)) + .roles(Set.of()) + .build(); + DiscoveryNode node2 = DiscoveryNodeUtils.builder("nodeId2") + .name("node2") + .ephemeralId("ephemeralId2") + .address("hostName2", "hostAddress2", new TransportAddress(TransportAddress.META_ADDRESS, 9500)) + .roles(Set.of()) + .build(); + Map metaMap1 = DeprecationIssue.createMetaMapForRemovableSettings(List.of("setting.1", "setting.2", "setting.3")); + Map metaMap2 = DeprecationIssue.createMetaMapForRemovableSettings(List.of("setting.2", "setting.3")); + DeprecationIssue foundIssue1 = createTestDeprecationIssue(metaMap1); + DeprecationIssue foundIssue2 = createTestDeprecationIssue(foundIssue1, metaMap2); + + NodesDeprecationCheckResponse nodeDeprecationIssues = new NodesDeprecationCheckResponse( + new ClusterName(randomAlphaOfLength(5)), + Arrays.asList( + new NodesDeprecationCheckAction.NodeResponse(node1, List.of(foundIssue1)), + new NodesDeprecationCheckAction.NodeResponse(node2, List.of(foundIssue2)) + ), + List.of() + ); + + List result = NodeDeprecationChecker.reduceToDeprecationIssues(nodeDeprecationIssues); + + String details = foundIssue1.getDetails() != null ? foundIssue1.getDetails() + " " : ""; + DeprecationIssue mergedFoundIssue = new DeprecationIssue( + foundIssue1.getLevel(), + foundIssue1.getMessage(), + foundIssue1.getUrl(), + details + "(nodes impacted: [" + node1.getName() + ", " + node2.getName() + "])", + foundIssue1.isResolveDuringRollingUpgrade(), + foundIssue2.getMeta() + ); + assertThat(result, equalTo(List.of(mergedFoundIssue))); + } +} diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java index 3aaee0e5cdb52..18b85ff532234 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java @@ -30,9 +30,11 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.function.Function; import java.util.stream.Collectors; -import static org.elasticsearch.xpack.deprecation.DeprecationChecks.NODE_SETTINGS_CHECKS; +import static org.elasticsearch.xpack.deprecation.NodeDeprecationChecks.SINGLE_NODE_CHECKS; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.not; @@ -154,8 +156,8 @@ public void testSharedDataPathSetting() { .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), createTempDir()) .build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); final String expectedUrl = @@ -209,8 +211,8 @@ public void testCheckReservedPrefixedRealmNames() { } final Settings settings = builder.build(); - final List deprecationIssues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + final List deprecationIssues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -235,8 +237,8 @@ public void testCheckReservedPrefixedRealmNames() { void monitoringSetting(String settingKey, String value) { Settings settings = Settings.builder().put(settingKey, value).build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); final String expectedUrl = "https://ela.st/es-deprecation-7-monitoring-settings"; @@ -259,8 +261,8 @@ void monitoringExporterSetting(String suffix, String value) { String settingKey = "xpack.monitoring.exporters.test." + suffix; Settings settings = Settings.builder().put(settingKey, value).build(); final XPackLicenseState licenseState = new XPackLicenseState(() -> 0); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, licenseState) ); final String expectedUrl = "https://ela.st/es-deprecation-7-monitoring-settings"; @@ -284,8 +286,8 @@ void monitoringExporterGroupedSetting(String suffix, String value) { String subSettingKey = settingKey + ".subsetting"; Settings settings = Settings.builder().put(subSettingKey, value).build(); final XPackLicenseState licenseState = new XPackLicenseState(() -> 0); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, licenseState) ); final String expectedUrl = "https://ela.st/es-deprecation-7-monitoring-settings"; @@ -310,8 +312,8 @@ void monitoringExporterSecureSetting(String suffix, String value) { secureSettings.setString(settingKey, value); Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); final XPackLicenseState licenseState = new XPackLicenseState(() -> 0); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, licenseState) ); final String expectedUrl = "https://ela.st/es-deprecation-7-monitoring-settings"; @@ -457,8 +459,8 @@ public void testCheckMonitoringSettingCollectionInterval() { public void testExporterUseIngestPipelineSettings() { Settings settings = Settings.builder().put("xpack.monitoring.exporters.test.use_ingest", true).build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -483,8 +485,8 @@ public void testExporterPipelineMasterTimeoutSetting() { .put("xpack.monitoring.exporters.test.index.pipeline.master_timeout", TimeValue.timeValueSeconds(10)) .build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -508,8 +510,8 @@ public void testExporterPipelineMasterTimeoutSetting() { public void testExporterCreateLegacyTemplateSetting() { Settings settings = Settings.builder().put("xpack.monitoring.exporters.test.index.template.create_legacy_templates", true).build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -535,8 +537,8 @@ public void testScriptContextCacheSetting() { .put(ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), "use-context") .build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -564,8 +566,8 @@ public void testScriptContextCompilationsRateLimitSetting() { .put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(contexts.get(1)).getKey(), "456/7m") .build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -601,8 +603,8 @@ public void testImplicitScriptContextCacheSetting() { .put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(contexts.get(1)).getKey(), "2453") .build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -639,8 +641,8 @@ public void testScriptContextCacheSizeSetting() { .put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(contexts.get(1)).getKey(), 200) .build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -676,8 +678,8 @@ public void testScriptContextCacheExpirationSetting() { .put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(contexts.get(1)).getKey(), "2d") .build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -708,8 +710,8 @@ public void testScriptContextCacheExpirationSetting() { public void testEnforceDefaultTierPreferenceSetting() { Settings settings = Settings.builder().put(DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING.getKey(), randomBoolean()).build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -731,8 +733,8 @@ public void testEnforceDefaultTierPreferenceSetting() { } private List getDeprecationIssues(Settings settings, PluginsAndModules pluginsAndModules) { - final List issues = DeprecationChecks.filterChecks( - DeprecationChecks.NODE_SETTINGS_CHECKS, + final List issues = filterChecks( + NodeDeprecationChecks.SINGLE_NODE_CHECKS, c -> c.apply(settings, pluginsAndModules, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -799,8 +801,8 @@ public void testDynamicSettings() { } Metadata metadata = metadataBuilder.build(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); - final List issues = DeprecationChecks.filterChecks( - DeprecationChecks.NODE_SETTINGS_CHECKS, + final List issues = filterChecks( + NodeDeprecationChecks.SINGLE_NODE_CHECKS, c -> c.apply(nodettings, pluginsAndModules, clusterState, licenseState) ); @@ -832,4 +834,8 @@ public void testCheckNodeAttrData() { ); assertThat(issues, hasItem(expected)); } + + static List filterChecks(List checks, Function mapper) { + return checks.stream().map(mapper).filter(Objects::nonNull).toList(); + } } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationCheckerTests.java index 81c0d1c7dc918..69bb396a50a91 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationCheckerTests.java @@ -48,7 +48,7 @@ public void testCheckSourceModeInComponentTemplates() throws IOException { ) .build(); - Map> issuesByComponentTemplate = checker.check(clusterState, null); + Map> issuesByComponentTemplate = checker.check(clusterState); final DeprecationIssue expected = new DeprecationIssue( DeprecationIssue.Level.CRITICAL, SourceFieldMapper.DEPRECATION_WARNING, @@ -81,7 +81,7 @@ public void testCheckLegacyTiersInComponentTemplates() { ) .build(); - Map> issuesByComponentTemplate = checker.check(clusterState, null); + Map> issuesByComponentTemplate = checker.check(clusterState); final DeprecationIssue expected = new DeprecationIssue( DeprecationIssue.Level.WARNING, "Configuring tiers via filtered allocation is not recommended.", @@ -121,7 +121,7 @@ public void testCheckLegacyTierSettings() { ) .build(); - Map> issuesByComponentTemplate = checker.check(clusterState, null); + Map> issuesByComponentTemplate = checker.check(clusterState); final DeprecationIssue expected = new DeprecationIssue( DeprecationIssue.Level.WARNING, "Configuring tiers via filtered allocation is not recommended.", @@ -164,7 +164,7 @@ public void testComponentAndComposableTemplateWithSameName() { ) .build(); - Map> issuesByComponentTemplate = checker.check(clusterState, null); + Map> issuesByComponentTemplate = checker.check(clusterState); final DeprecationIssue expectedIndexTemplateIssue = new DeprecationIssue( DeprecationIssue.Level.WARNING, "Configuring tiers via filtered allocation is not recommended.", diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoActionTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoActionTests.java index 85fa375c09c5f..945068ba3a107 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoActionTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoActionTests.java @@ -6,22 +6,295 @@ */ package org.elasticsearch.xpack.deprecation; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; +import org.hamcrest.core.IsEqual; +import org.junit.Assert; +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.Response.RESERVED_NAMES; +import static org.elasticsearch.xpack.deprecation.DeprecationInfoActionResponseTests.createTestDeprecationIssue; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TransportDeprecationInfoActionTests extends ESTestCase { + public void testCheckAndCreateResponse() throws IOException { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all"); + mapping.field("enabled", false); + mapping.endObject().endObject(); + + Metadata metadata = Metadata.builder() + .put( + IndexMetadata.builder("test") + .putMapping(Strings.toString(mapping)) + .settings(settings(IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + ) + .build(); + + DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("test", new TransportAddress(TransportAddress.META_ADDRESS, 9300)); + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); + IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); + boolean clusterIssueFound = randomBoolean(); + boolean nodeIssueFound = randomBoolean(); + boolean indexIssueFound = randomBoolean(); + boolean dataStreamIssueFound = randomBoolean(); + boolean indexTemplateIssueFound = randomBoolean(); + boolean componentTemplateIssueFound = randomBoolean(); + boolean ilmPolicyIssueFound = randomBoolean(); + DeprecationIssue foundIssue = createTestDeprecationIssue(); + ClusterDeprecationChecker clusterDeprecationChecker = mock(ClusterDeprecationChecker.class); + when(clusterDeprecationChecker.check(any(), any())).thenReturn(clusterIssueFound ? List.of(foundIssue) : List.of()); + List resourceCheckers = List.of(createResourceChecker("index_settings", (cs, req) -> { + if (indexIssueFound) { + return Map.of("test", List.of(foundIssue)); + } + return Map.of(); + }), createResourceChecker("data_streams", (cs, req) -> { + if (dataStreamIssueFound) { + return Map.of("my-ds", List.of(foundIssue)); + } + return Map.of(); + }), createResourceChecker("templates", (cs, req) -> { + Map> issues = new HashMap<>(); + if (componentTemplateIssueFound) { + issues.put("my-component-template", List.of(foundIssue)); + } + if (indexTemplateIssueFound) { + issues.put("my-index-template", List.of(foundIssue)); + } + return issues; + }), createResourceChecker("ilm_policies", (cs, req) -> { + if (ilmPolicyIssueFound) { + return Map.of("my-policy", List.of(foundIssue)); + } + return Map.of(); + })); + + List nodeDeprecationIssues = nodeIssueFound ? List.of(foundIssue) : List.of(); + + DeprecationInfoAction.Request request = new DeprecationInfoAction.Request(randomTimeValue(), Strings.EMPTY_ARRAY); + TransportDeprecationInfoAction.PrecomputedData precomputedData = new TransportDeprecationInfoAction.PrecomputedData(); + precomputedData.setOnceTransformConfigs(List.of()); + precomputedData.setOncePluginIssues(Map.of()); + precomputedData.setOnceNodeSettingsIssues(nodeDeprecationIssues); + DeprecationInfoAction.Response response = TransportDeprecationInfoAction.checkAndCreateResponse( + state, + resolver, + request, + List.of(), + clusterDeprecationChecker, + resourceCheckers, + precomputedData + ); + + if (clusterIssueFound) { + assertThat(response.getClusterSettingsIssues(), IsEqual.equalTo(List.of(foundIssue))); + } else { + assertThat(response.getClusterSettingsIssues(), empty()); + } + + if (nodeIssueFound) { + assertThat(response.getNodeSettingsIssues(), IsEqual.equalTo(List.of(foundIssue))); + } else { + assertTrue(response.getNodeSettingsIssues().isEmpty()); + } + + if (indexIssueFound) { + assertThat(response.getIndexSettingsIssues(), IsEqual.equalTo(Map.of("test", List.of(foundIssue)))); + } else { + assertTrue(response.getIndexSettingsIssues().isEmpty()); + } + if (dataStreamIssueFound) { + assertThat(response.getDataStreamDeprecationIssues(), IsEqual.equalTo(Map.of("my-ds", List.of(foundIssue)))); + } else { + assertTrue(response.getDataStreamDeprecationIssues().isEmpty()); + } + if (ilmPolicyIssueFound) { + assertThat(response.getIlmPolicyDeprecationIssues(), IsEqual.equalTo(Map.of("my-policy", List.of(foundIssue)))); + } else { + assertTrue(response.getIlmPolicyDeprecationIssues().isEmpty()); + } + if (componentTemplateIssueFound == false && indexTemplateIssueFound == false) { + assertTrue(response.getTemplateDeprecationIssues().isEmpty()); + } else { + if (componentTemplateIssueFound) { + assertThat(response.getTemplateDeprecationIssues().get("my-component-template"), IsEqual.equalTo(List.of(foundIssue))); + } + if (indexTemplateIssueFound) { + assertThat(response.getTemplateDeprecationIssues().get("my-index-template"), IsEqual.equalTo(List.of(foundIssue))); + } + + } + } + + public void testRemoveSkippedSettings() { + Settings.Builder settingsBuilder = settings(IndexVersion.current()); + settingsBuilder.put("some.deprecated.property", "someValue1"); + settingsBuilder.put("some.other.bad.deprecated.property", "someValue2"); + settingsBuilder.put("some.undeprecated.property", "someValue3"); + settingsBuilder.putList("some.undeprecated.list.property", List.of("someValue4", "someValue5")); + Settings inputSettings = settingsBuilder.build(); + IndexMetadata dataStreamIndexMetadata = IndexMetadata.builder("ds-test-index-1") + .settings(inputSettings) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + ComponentTemplate componentTemplate = new ComponentTemplate(Template.builder().settings(inputSettings).build(), null, null); + ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() + .template(Template.builder().settings(inputSettings)) + .build(); + Metadata metadata = Metadata.builder() + .put(IndexMetadata.builder("test").settings(inputSettings).numberOfShards(1).numberOfReplicas(0)) + .put(dataStreamIndexMetadata, true) + .put(DataStream.builder("ds-test", List.of(dataStreamIndexMetadata.getIndex())).build()) + .indexTemplates( + Map.of( + "my-index-template", + indexTemplate, + "empty-template", + ComposableIndexTemplate.builder().indexPatterns(List.of("random")).build() + ) + ) + .componentTemplates(Map.of("my-component-template", componentTemplate)) + .persistentSettings(inputSettings) + .build(); + + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); + IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); + AtomicReference visibleClusterSettings = new AtomicReference<>(); + ClusterDeprecationChecker clusterDeprecationChecker = mock(ClusterDeprecationChecker.class); + when(clusterDeprecationChecker.check(any(), any())).thenAnswer(invocationOnMock -> { + ClusterState observedState = invocationOnMock.getArgument(0); + visibleClusterSettings.set(observedState.getMetadata().settings()); + return List.of(); + }); + AtomicReference visibleIndexSettings = new AtomicReference<>(); + AtomicReference visibleComponentTemplateSettings = new AtomicReference<>(); + AtomicReference visibleIndexTemplateSettings = new AtomicReference<>(); + AtomicInteger backingIndicesCount = new AtomicInteger(0); + List resourceCheckers = List.of(createResourceChecker("index_settings", (cs, req) -> { + for (String indexName : resolver.concreteIndexNames(cs, req)) { + visibleIndexSettings.set(cs.metadata().index(indexName).getSettings()); + } + return Map.of(); + }), createResourceChecker("data_streams", (cs, req) -> { + cs.metadata().dataStreams().values().forEach(ds -> backingIndicesCount.set(ds.getIndices().size())); + return Map.of(); + }), createResourceChecker("templates", (cs, req) -> { + cs.metadata() + .componentTemplates() + .values() + .forEach(template -> visibleComponentTemplateSettings.set(template.template().settings())); + cs.metadata().templatesV2().values().forEach(template -> { + if (template.template() != null && template.template().settings() != null) { + visibleIndexTemplateSettings.set(template.template().settings()); + } + }); + return Map.of(); + })); + TransportDeprecationInfoAction.PrecomputedData precomputedData = new TransportDeprecationInfoAction.PrecomputedData(); + precomputedData.setOnceTransformConfigs(List.of()); + precomputedData.setOncePluginIssues(Map.of()); + precomputedData.setOnceNodeSettingsIssues(List.of()); + DeprecationInfoAction.Request request = new DeprecationInfoAction.Request(randomTimeValue(), Strings.EMPTY_ARRAY); + TransportDeprecationInfoAction.checkAndCreateResponse( + state, + resolver, + request, + List.of("some.deprecated.property", "some.other.*.deprecated.property"), + clusterDeprecationChecker, + resourceCheckers, + precomputedData + ); + + settingsBuilder = settings(IndexVersion.current()); + settingsBuilder.put("some.undeprecated.property", "someValue3"); + settingsBuilder.putList("some.undeprecated.list.property", List.of("someValue4", "someValue5")); + + Settings expectedSettings = settingsBuilder.build(); + Settings resultClusterSettings = visibleClusterSettings.get(); + Assert.assertNotNull(resultClusterSettings); + Assert.assertEquals(expectedSettings, visibleClusterSettings.get()); + + Settings resultIndexSettings = visibleIndexSettings.get(); + Assert.assertNotNull(resultIndexSettings); + Assert.assertEquals("someValue3", resultIndexSettings.get("some.undeprecated.property")); + Assert.assertEquals(resultIndexSettings.getAsList("some.undeprecated.list.property"), List.of("someValue4", "someValue5")); + Assert.assertFalse(resultIndexSettings.hasValue("some.deprecated.property")); + Assert.assertFalse(resultIndexSettings.hasValue("some.other.bad.deprecated.property")); + + assertThat(backingIndicesCount.get(), IsEqual.equalTo(1)); + + Assert.assertNotNull(visibleComponentTemplateSettings.get()); + Assert.assertEquals(expectedSettings, visibleComponentTemplateSettings.get()); + Assert.assertNotNull(visibleIndexTemplateSettings.get()); + Assert.assertEquals(expectedSettings, visibleIndexTemplateSettings.get()); + } + + public void testCtorFailure() { + Map> indexNames = Stream.generate(() -> randomAlphaOfLength(10)) + .limit(10) + .collect(Collectors.toMap(Function.identity(), (_k) -> List.of())); + Map> dataStreamNames = Stream.generate(() -> randomAlphaOfLength(10)) + .limit(10) + .collect(Collectors.toMap(Function.identity(), (_k) -> List.of())); + Set shouldCauseFailure = new HashSet<>(RESERVED_NAMES); + for (int i = 0; i < randomIntBetween(1, 100); i++) { + Map> pluginSettingsIssues = randomSubsetOf(3, shouldCauseFailure).stream() + .collect(Collectors.toMap(Function.identity(), (_k) -> List.of())); + expectThrows( + ElasticsearchStatusException.class, + () -> new DeprecationInfoAction.Response( + List.of(), + List.of(), + Map.of("data_streams", dataStreamNames, "index_settings", indexNames), + pluginSettingsIssues + ) + ); + } + } + public void testPluginSettingIssues() { DeprecationChecker.Components components = new DeprecationChecker.Components(null, Settings.EMPTY, null); PlainActionFuture>> future = new PlainActionFuture<>(); @@ -65,6 +338,28 @@ public void testPluginSettingIssuesWithFailures() { assertThat(exception.getCause().getMessage(), containsString("boom")); } + private static ResourceDeprecationChecker createResourceChecker( + String name, + BiFunction>> check + ) { + return new ResourceDeprecationChecker() { + + @Override + public Map> check( + ClusterState clusterState, + DeprecationInfoAction.Request request, + TransportDeprecationInfoAction.PrecomputedData precomputedData + ) { + return check.apply(clusterState, request); + } + + @Override + public String getName() { + return name; + } + }; + } + private static class NamedChecker implements DeprecationChecker { private final String name; diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java index 80692efb7474a..a0a37f2bb52d1 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java @@ -61,7 +61,7 @@ public void testNodeOperation() { settingsBuilder.put("some.undeprecated.property", "someValue3"); settingsBuilder.putList("some.undeprecated.list.property", List.of("someValue4", "someValue5")); settingsBuilder.putList( - DeprecationChecks.SKIP_DEPRECATIONS_SETTING.getKey(), + TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING.getKey(), List.of("some.deprecated.property", "some.other.*.deprecated.property", "some.bad.dynamic.property") ); Settings nodeSettings = settingsBuilder.build(); @@ -73,7 +73,10 @@ public void testNodeOperation() { ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); ClusterService clusterService = Mockito.mock(ClusterService.class); when(clusterService.state()).thenReturn(clusterState); - ClusterSettings clusterSettings = new ClusterSettings(nodeSettings, Set.of(DeprecationChecks.SKIP_DEPRECATIONS_SETTING)); + ClusterSettings clusterSettings = new ClusterSettings( + nodeSettings, + Set.of(TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING) + ); when((clusterService.getClusterSettings())).thenReturn(clusterSettings); DiscoveryNode node = Mockito.mock(DiscoveryNode.class); when(node.getId()).thenReturn("mock-node"); @@ -98,7 +101,7 @@ public void testNodeOperation() { NodesDeprecationCheckAction.NodeRequest nodeRequest = null; AtomicReference visibleNodeSettings = new AtomicReference<>(); AtomicReference visibleClusterStateMetadataSettings = new AtomicReference<>(); - DeprecationChecks.NodeDeprecationCheck< + NodeDeprecationChecks.NodeDeprecationCheck< Settings, PluginsAndModules, ClusterState, @@ -109,7 +112,7 @@ public void testNodeOperation() { return null; }; java.util.List< - DeprecationChecks.NodeDeprecationCheck< + NodeDeprecationChecks.NodeDeprecationCheck< Settings, PluginsAndModules, ClusterState, @@ -120,7 +123,7 @@ public void testNodeOperation() { settingsBuilder.put("some.undeprecated.property", "someValue3"); settingsBuilder.putList("some.undeprecated.list.property", List.of("someValue4", "someValue5")); settingsBuilder.putList( - DeprecationChecks.SKIP_DEPRECATIONS_SETTING.getKey(), + TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING.getKey(), List.of("some.deprecated.property", "some.other.*.deprecated.property", "some.bad.dynamic.property") ); Settings expectedSettings = settingsBuilder.build(); @@ -131,7 +134,7 @@ public void testNodeOperation() { // Testing that the setting is dynamically updatable: Settings newSettings = Settings.builder() - .putList(DeprecationChecks.SKIP_DEPRECATIONS_SETTING.getKey(), List.of("some.undeprecated.property")) + .putList(TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING.getKey(), List.of("some.undeprecated.property")) .build(); clusterSettings.applySettings(newSettings); transportNodeDeprecationCheckAction.nodeOperation(nodeRequest, nodeSettingsChecks); @@ -141,7 +144,7 @@ public void testNodeOperation() { settingsBuilder.putList("some.undeprecated.list.property", List.of("someValue4", "someValue5")); // This is the node setting (since this is the node deprecation check), not the cluster setting: settingsBuilder.putList( - DeprecationChecks.SKIP_DEPRECATIONS_SETTING.getKey(), + TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING.getKey(), List.of("some.deprecated.property", "some.other.*.deprecated.property", "some.bad.dynamic.property") ); expectedSettings = settingsBuilder.build();