diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java index bd244ff9638ff..ba333e09d2859 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java @@ -490,20 +490,7 @@ public void testProvidersAffectMode() { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, - new IndexSettingProviders( - Set.of( - ( - indexName, - dataStreamName, - templateIndexMode, - metadata, - resolvedAt, - indexTemplateAndCreateRequestSettings, - combinedTemplateMappings, - additionalSettings, - additionalCustomMetadata) -> additionalSettings.put("index.mode", IndexMode.LOOKUP) - ) - ), + IndexSettingProviders.of((additionalSettings) -> additionalSettings.put("index.mode", IndexMode.LOOKUP)), null ); assertThat(response.getDataStreams().getFirst().getIndexModeName(), equalTo("lookup")); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java index e8bd38b01414f..2bc51b252386c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java @@ -52,6 +52,8 @@ public class CreateIndexClusterStateUpdateRequest { private ComposableIndexTemplate matchingTemplate; + private boolean settingsSystemProvided = false; + /** * @deprecated project id ought always be specified */ @@ -223,6 +225,19 @@ public CreateIndexClusterStateUpdateRequest setMatchingTemplate(ComposableIndexT return this; } + /** + * Indicates whether the {@link #settings} of this request are system provided. + * If this is true, private settings will be allowed to be set in the request. + */ + public CreateIndexClusterStateUpdateRequest settingsSystemProvided(boolean settingsSystemProvided) { + this.settingsSystemProvided = settingsSystemProvided; + return this; + } + + public boolean settingsSystemProvided() { + return settingsSystemProvided; + } + @Override public String toString() { return "CreateIndexClusterStateUpdateRequest{" diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index cdb43e865cae4..134093b5175b5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -1607,12 +1607,12 @@ private static void validateActiveShardCount(ActiveShardCount waitForActiveShard private void validate(CreateIndexClusterStateUpdateRequest request, ProjectMetadata projectMetadata, RoutingTable routingTable) { validateIndexName(request.index(), projectMetadata, routingTable); - validateIndexSettings(request.index(), request.settings(), forbidPrivateIndexSettings); + validateIndexSettings(request.index(), request.settings(), forbidPrivateIndexSettings && request.settingsSystemProvided() == false); } public void validateIndexSettings(String indexName, final Settings settings, final boolean forbidPrivateIndexSettings) throws IndexCreationException { - List validationErrors = getIndexSettingsValidationErrors(settings, forbidPrivateIndexSettings); + List validationErrors = getIndexSettingsValidationErrors(settings, null, forbidPrivateIndexSettings); if (validationErrors.isEmpty() == false) { ValidationException validationException = new ValidationException(); @@ -1621,23 +1621,34 @@ public void validateIndexSettings(String indexName, final Settings settings, fin } } - List getIndexSettingsValidationErrors(final Settings settings, final boolean forbidPrivateIndexSettings) { + List getIndexSettingsValidationErrors( + final Settings settings, + @Nullable Settings systemProvided, + final boolean forbidPrivateIndexSettings + ) { List validationErrors = validateIndexCustomPath(settings, env.sharedDataDir()); if (forbidPrivateIndexSettings) { - validationErrors.addAll(validatePrivateSettingsNotExplicitlySet(settings, indexScopedSettings)); + validationErrors.addAll(validatePrivateSettingsNotExplicitlySet(settings, systemProvided, indexScopedSettings)); } return validationErrors; } - private static List validatePrivateSettingsNotExplicitlySet(Settings settings, IndexScopedSettings indexScopedSettings) { + private static List validatePrivateSettingsNotExplicitlySet( + Settings settings, + @Nullable Settings systemProvided, + IndexScopedSettings indexScopedSettings + ) { List validationErrors = new ArrayList<>(); for (final String key : settings.keySet()) { final Setting setting = indexScopedSettings.get(key); if (setting == null) { assert indexScopedSettings.isPrivateSetting(key) : "expected [" + key + "] to be private but it was not"; - } else if (setting.isPrivateIndex()) { - validationErrors.add("private index setting [" + key + "] can not be set explicitly"); - } + } else if (setting.isPrivateIndex() + // System-provided settings are always allowed to configure private settings. + // These are typically coming from an IndexSettingProvider. + && (systemProvided == null || settings.get(key).equals(systemProvided.get(key)) == false)) { + validationErrors.add("private index setting [" + key + "] can not be set explicitly"); + } } return validationErrors; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index a3dbd34605c18..08b1773fa2c17 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -788,8 +788,7 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo final var combinedMappings = collectMappings(indexTemplate, projectMetadata.componentTemplates(), "tmp_idx"); final var combinedSettings = resolveSettings(indexTemplate, projectMetadata.componentTemplates()); - // First apply settings sourced from index setting providers: - var finalSettings = Settings.builder(); + var additionalSettingsBuilder = Settings.builder(); ImmutableOpenMap.Builder> customMetadataBuilder = ImmutableOpenMap.builder(); for (var provider : indexSettingProviders) { Settings.Builder builder = Settings.builder(); @@ -805,9 +804,13 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo customMetadataBuilder::put ); var newAdditionalSettings = builder.build(); - MetadataCreateIndexService.validateAdditionalSettings(provider, newAdditionalSettings, finalSettings); - finalSettings.put(newAdditionalSettings); + MetadataCreateIndexService.validateAdditionalSettings(provider, newAdditionalSettings, additionalSettingsBuilder); + additionalSettingsBuilder.put(newAdditionalSettings); } + Settings additionalSettings = additionalSettingsBuilder.build(); + var finalSettings = Settings.builder(); + // First apply settings sourced from index setting providers: + finalSettings.put(additionalSettings); // Then apply setting from component templates: finalSettings.put(combinedSettings); // Then finally apply settings resolved from index template: @@ -817,7 +820,7 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo var templateToValidate = indexTemplate.toBuilder().template(Template.builder(finalTemplate).settings(finalSettings)).build(); - validate(name, templateToValidate); + validate(name, templateToValidate, additionalSettings); validateDataStreamsStillReferenced(projectMetadata, name, templateToValidate); validateLifecycle(projectMetadata, name, templateToValidate, globalRetentionSettings.get(false)); validateDataStreamOptions(projectMetadata, name, templateToValidate, globalRetentionSettings.get(true)); @@ -2057,18 +2060,19 @@ public static void validateTemplate(Settings validateSettings, CompressedXConten } public void validate(String name, ComponentTemplate template) { - validate(name, template.template(), Collections.emptyList()); + validate(name, template.template(), Collections.emptyList(), null); } - private void validate(String name, ComposableIndexTemplate template) { - validate(name, template.template(), template.indexPatterns()); + private void validate(String name, ComposableIndexTemplate template, @Nullable Settings systemProvided) { + validate(name, template.template(), template.indexPatterns(), systemProvided); } - private void validate(String name, Template template, List indexPatterns) { + private void validate(String name, Template template, List indexPatterns, @Nullable Settings systemProvided) { Optional