Skip to content

Commit 1b765de

Browse files
felixbarnyfelixbarny
authored
Allow private settings when system provided (#133789)
1 parent 038a4f2 commit 1b765de

File tree

8 files changed

+253
-36
lines changed

8 files changed

+253
-36
lines changed

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java‎

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -490,20 +490,7 @@ public void testProvidersAffectMode() {
490490
ClusterSettings.createBuiltInClusterSettings(),
491491
dataStreamGlobalRetentionSettings,
492492
emptyDataStreamFailureStoreSettings,
493-
new IndexSettingProviders(
494-
Set.of(
495-
(
496-
indexName,
497-
dataStreamName,
498-
templateIndexMode,
499-
metadata,
500-
resolvedAt,
501-
indexTemplateAndCreateRequestSettings,
502-
combinedTemplateMappings,
503-
additionalSettings,
504-
additionalCustomMetadata) -> additionalSettings.put("index.mode", IndexMode.LOOKUP)
505-
)
506-
),
493+
IndexSettingProviders.of((additionalSettings) -> additionalSettings.put("index.mode", IndexMode.LOOKUP)),
507494
null
508495
);
509496
assertThat(response.getDataStreams().getFirst().getIndexModeName(), equalTo("lookup"));

‎server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import org.elasticsearch.cluster.metadata.IndexMetadata;
1717
import org.elasticsearch.cluster.metadata.Metadata;
1818
import org.elasticsearch.cluster.metadata.ProjectId;
19+
import org.elasticsearch.common.settings.Setting;
1920
import org.elasticsearch.common.settings.Settings;
2021
import org.elasticsearch.index.Index;
2122
import org.elasticsearch.indices.SystemDataStreamDescriptor;
@@ -52,6 +53,8 @@ public class CreateIndexClusterStateUpdateRequest {
5253

5354
private ComposableIndexTemplate matchingTemplate;
5455

56+
private boolean settingsSystemProvided = false;
57+
5558
/**
5659
* @deprecated project id ought always be specified
5760
*/
@@ -223,6 +226,20 @@ public CreateIndexClusterStateUpdateRequest setMatchingTemplate(ComposableIndexT
223226
return this;
224227
}
225228

229+
/**
230+
* Indicates whether the {@link #settings} of this request are system provided.
231+
* System-provided settings are allowed to configure {@linkplain Setting.Property#PrivateIndex private} settings.
232+
* These are typically coming from an {@link org.elasticsearch.index.IndexSettingProvider}.
233+
*/
234+
public CreateIndexClusterStateUpdateRequest settingsSystemProvided(boolean settingsSystemProvided) {
235+
this.settingsSystemProvided = settingsSystemProvided;
236+
return this;
237+
}
238+
239+
public boolean settingsSystemProvided() {
240+
return settingsSystemProvided;
241+
}
242+
226243
@Override
227244
public String toString() {
228245
return "CreateIndexClusterStateUpdateRequest{"

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1607,12 +1607,12 @@ private static void validateActiveShardCount(ActiveShardCount waitForActiveShard
16071607

16081608
private void validate(CreateIndexClusterStateUpdateRequest request, ProjectMetadata projectMetadata, RoutingTable routingTable) {
16091609
validateIndexName(request.index(), projectMetadata, routingTable);
1610-
validateIndexSettings(request.index(), request.settings(), forbidPrivateIndexSettings);
1610+
validateIndexSettings(request.index(), request.settings(), forbidPrivateIndexSettings && request.settingsSystemProvided() == false);
16111611
}
16121612

16131613
public void validateIndexSettings(String indexName, final Settings settings, final boolean forbidPrivateIndexSettings)
16141614
throws IndexCreationException {
1615-
List<String> validationErrors = getIndexSettingsValidationErrors(settings, forbidPrivateIndexSettings);
1615+
List<String> validationErrors = getIndexSettingsValidationErrors(settings, null, forbidPrivateIndexSettings);
16161616

16171617
if (validationErrors.isEmpty() == false) {
16181618
ValidationException validationException = new ValidationException();
@@ -1621,27 +1621,43 @@ public void validateIndexSettings(String indexName, final Settings settings, fin
16211621
}
16221622
}
16231623

1624-
List<String> getIndexSettingsValidationErrors(final Settings settings, final boolean forbidPrivateIndexSettings) {
1624+
List<String> getIndexSettingsValidationErrors(
1625+
final Settings settings,
1626+
@Nullable Settings systemProvided,
1627+
final boolean forbidPrivateIndexSettings
1628+
) {
16251629
List<String> validationErrors = validateIndexCustomPath(settings, env.sharedDataDir());
16261630
if (forbidPrivateIndexSettings) {
1627-
validationErrors.addAll(validatePrivateSettingsNotExplicitlySet(settings, indexScopedSettings));
1631+
validationErrors.addAll(validatePrivateSettingsNotExplicitlySet(settings, systemProvided, indexScopedSettings));
16281632
}
16291633
return validationErrors;
16301634
}
16311635

1632-
private static List<String> validatePrivateSettingsNotExplicitlySet(Settings settings, IndexScopedSettings indexScopedSettings) {
1636+
private static List<String> validatePrivateSettingsNotExplicitlySet(
1637+
Settings settings,
1638+
@Nullable Settings systemProvided,
1639+
IndexScopedSettings indexScopedSettings
1640+
) {
16331641
List<String> validationErrors = new ArrayList<>();
16341642
for (final String key : settings.keySet()) {
16351643
final Setting<?> setting = indexScopedSettings.get(key);
16361644
if (setting == null) {
16371645
assert indexScopedSettings.isPrivateSetting(key) : "expected [" + key + "] to be private but it was not";
1638-
} else if (setting.isPrivateIndex()) {
1646+
} else if (setting.isPrivateIndex() && isSystemProvided(key, settings, systemProvided) == false) {
16391647
validationErrors.add("private index setting [" + key + "] can not be set explicitly");
16401648
}
16411649
}
16421650
return validationErrors;
16431651
}
16441652

1653+
/*
1654+
* System-provided settings are always allowed to configure private settings.
1655+
* These are typically coming from an IndexSettingProvider.
1656+
*/
1657+
private static boolean isSystemProvided(String key, Settings settings, @Nullable Settings systemProvided) {
1658+
return systemProvided != null && settings.get(key).equals(systemProvided.get(key));
1659+
}
1660+
16451661
/**
16461662
* Validates that the configured index data path (if any) is a sub-path of the configured shared data path (if any)
16471663
*

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java‎

Lines changed: 26 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -786,8 +786,7 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo
786786

787787
final var combinedMappings = collectMappings(indexTemplate, projectMetadata.componentTemplates(), "tmp_idx");
788788
final var combinedSettings = resolveSettings(indexTemplate, projectMetadata.componentTemplates());
789-
// First apply settings sourced from index setting providers:
790-
var finalSettings = Settings.builder();
789+
var additionalSettingsBuilder = Settings.builder();
791790
ImmutableOpenMap.Builder<String, Map<String, String>> customMetadataBuilder = ImmutableOpenMap.builder();
792791
for (var provider : indexSettingProviders) {
793792
Settings.Builder builder = Settings.builder();
@@ -803,9 +802,13 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo
803802
customMetadataBuilder::put
804803
);
805804
var newAdditionalSettings = builder.build();
806-
MetadataCreateIndexService.validateAdditionalSettings(provider, newAdditionalSettings, finalSettings);
807-
finalSettings.put(newAdditionalSettings);
805+
MetadataCreateIndexService.validateAdditionalSettings(provider, newAdditionalSettings, additionalSettingsBuilder);
806+
additionalSettingsBuilder.put(newAdditionalSettings);
808807
}
808+
Settings additionalSettings = additionalSettingsBuilder.build();
809+
var finalSettings = Settings.builder();
810+
// First apply settings sourced from index setting providers:
811+
finalSettings.put(additionalSettings);
809812
// Then apply setting from component templates:
810813
finalSettings.put(combinedSettings);
811814
// Then finally apply settings resolved from index template:
@@ -815,7 +818,7 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo
815818

816819
var templateToValidate = indexTemplate.toBuilder().template(Template.builder(finalTemplate).settings(finalSettings)).build();
817820

818-
validate(name, templateToValidate);
821+
validate(name, templateToValidate, additionalSettings);
819822
validateDataStreamsStillReferenced(projectMetadata, name, templateToValidate);
820823
validateLifecycle(projectMetadata, name, templateToValidate, globalRetentionSettings.get(false));
821824
validateDataStreamOptions(projectMetadata, name, templateToValidate, globalRetentionSettings.get(true));
@@ -2059,18 +2062,19 @@ public static void validateTemplate(Settings validateSettings, CompressedXConten
20592062
}
20602063

20612064
public void validate(String name, ComponentTemplate template) {
2062-
validate(name, template.template(), Collections.emptyList());
2065+
validate(name, template.template(), Collections.emptyList(), null);
20632066
}
20642067

2065-
private void validate(String name, ComposableIndexTemplate template) {
2066-
validate(name, template.template(), template.indexPatterns());
2068+
private void validate(String name, ComposableIndexTemplate template, @Nullable Settings systemProvided) {
2069+
validate(name, template.template(), template.indexPatterns(), systemProvided);
20672070
}
20682071

2069-
private void validate(String name, Template template, List<String> indexPatterns) {
2072+
private void validate(String name, Template template, List<String> indexPatterns, @Nullable Settings systemProvided) {
20702073
Optional<Template> maybeTemplate = Optional.ofNullable(template);
20712074
validate(
20722075
name,
20732076
maybeTemplate.map(Template::settings).orElse(Settings.EMPTY),
2077+
systemProvided,
20742078
indexPatterns,
20752079
maybeTemplate.map(Template::aliases).orElse(emptyMap()).values().stream().map(MetadataIndexTemplateService::toAlias).toList()
20762080
);
@@ -2089,10 +2093,16 @@ private static Alias toAlias(AliasMetadata aliasMeta) {
20892093
}
20902094

20912095
private void validate(PutRequest putRequest) {
2092-
validate(putRequest.name, putRequest.settings, putRequest.indexPatterns, putRequest.aliases);
2096+
validate(putRequest.name, putRequest.settings, null, putRequest.indexPatterns, putRequest.aliases);
20932097
}
20942098

2095-
private void validate(String name, @Nullable Settings settings, List<String> indexPatterns, List<Alias> aliases) {
2099+
private void validate(
2100+
String name,
2101+
@Nullable Settings settings,
2102+
@Nullable Settings systemProvided,
2103+
List<String> indexPatterns,
2104+
List<Alias> aliases
2105+
) {
20962106
List<String> validationErrors = new ArrayList<>();
20972107
if (name.contains(" ")) {
20982108
validationErrors.add("name must not contain a space");
@@ -2145,7 +2155,11 @@ private void validate(String name, @Nullable Settings settings, List<String> ind
21452155
validationErrors.add(t.getMessage());
21462156
}
21472157
}
2148-
List<String> indexSettingsValidation = metadataCreateIndexService.getIndexSettingsValidationErrors(settings, true);
2158+
List<String> indexSettingsValidation = metadataCreateIndexService.getIndexSettingsValidationErrors(
2159+
settings,
2160+
systemProvided,
2161+
true
2162+
);
21492163
validationErrors.addAll(indexSettingsValidation);
21502164
}
21512165

‎server/src/main/java/org/elasticsearch/index/IndexSettingProviders.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,11 @@
99

1010
package org.elasticsearch.index;
1111

12+
import org.elasticsearch.common.settings.Settings;
13+
1214
import java.util.Collections;
1315
import java.util.Set;
16+
import java.util.function.Consumer;
1417

1518
/**
1619
* Keeps track of the {@link IndexSettingProvider} instances defined by plugins and
@@ -22,6 +25,31 @@ public final class IndexSettingProviders {
2225

2326
private final Set<IndexSettingProvider> indexSettingProviders;
2427

28+
/**
29+
* Utility method which creates an {@link IndexSettingProviders} instance that uses the provided consumer to add settings
30+
* to the index being created.
31+
* The primary use case is for tests that want to add specific settings without having to create a full implementation.
32+
*
33+
* @param settingsBuilderConsumer A consumer that adds index settings
34+
* @return An {@link IndexSettingProviders} instance that uses the provided consumer to add settings
35+
*/
36+
public static IndexSettingProviders of(Consumer<Settings.Builder> settingsBuilderConsumer) {
37+
return new IndexSettingProviders(
38+
Set.of(
39+
(
40+
indexName,
41+
dataStreamName,
42+
templateIndexMode,
43+
projectMetadata,
44+
resolvedAt,
45+
indexTemplateAndCreateRequestSettings,
46+
combinedTemplateMappings,
47+
additionalSettings,
48+
additionalCustomMetadata) -> settingsBuilderConsumer.accept(additionalSettings)
49+
)
50+
);
51+
}
52+
2553
public IndexSettingProviders(Set<IndexSettingProvider> indexSettingProviders) {
2654
this.indexSettingProviders = Collections.unmodifiableSet(indexSettingProviders);
2755
}

‎server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java‎

Lines changed: 106 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
import org.elasticsearch.index.shard.IndexLongFieldRange;
6565
import org.elasticsearch.indices.EmptySystemIndices;
6666
import org.elasticsearch.indices.IndexCreationException;
67+
import org.elasticsearch.indices.IndicesService;
6768
import org.elasticsearch.indices.InvalidAliasNameException;
6869
import org.elasticsearch.indices.InvalidIndexNameException;
6970
import org.elasticsearch.indices.ShardLimitValidator;
@@ -82,6 +83,7 @@
8283
import org.elasticsearch.xcontent.XContentFactory;
8384
import org.hamcrest.Matchers;
8485
import org.junit.Before;
86+
import org.mockito.ArgumentCaptor;
8587

8688
import java.io.IOException;
8789
import java.time.Instant;
@@ -128,6 +130,9 @@
128130
import static org.hamcrest.Matchers.notNullValue;
129131
import static org.hamcrest.Matchers.nullValue;
130132
import static org.hamcrest.Matchers.startsWith;
133+
import static org.mockito.ArgumentMatchers.any;
134+
import static org.mockito.Mockito.mock;
135+
import static org.mockito.Mockito.verify;
131136

132137
public class MetadataCreateIndexServiceTests extends ESTestCase {
133138

@@ -1739,6 +1744,106 @@ public void testCreateClusterBlocksTransformerForIndexCreation() {
17391744
);
17401745
}
17411746

1747+
public void testSetPrivateSettingsFails() throws Exception {
1748+
request.settings(Settings.builder().put(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID.getKey(), "private_setting").build());
1749+
1750+
IndicesService indicesService = mock(IndicesService.class);
1751+
withTemporaryClusterService(((clusterService, threadPool) -> {
1752+
MetadataCreateIndexService service = new MetadataCreateIndexService(
1753+
Settings.EMPTY,
1754+
clusterService,
1755+
indicesService,
1756+
null,
1757+
createTestShardLimitService(randomIntBetween(1, 1000), clusterService),
1758+
newEnvironment(),
1759+
new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS),
1760+
threadPool,
1761+
null,
1762+
EmptySystemIndices.INSTANCE,
1763+
true,
1764+
IndexSettingProviders.EMPTY
1765+
);
1766+
1767+
IndexCreationException exception = assertThrows(
1768+
IndexCreationException.class,
1769+
() -> service.applyCreateIndexRequest(clusterService.state(), request, false, ActionListener.wrap(r -> {}, e -> {}))
1770+
);
1771+
assertThat(
1772+
exception.getCause().getMessage(),
1773+
containsString(
1774+
"private index setting [" + IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID.getKey() + "] can not be set explicitly"
1775+
)
1776+
);
1777+
}));
1778+
}
1779+
1780+
public void testSetPrivateSettingsSucceedsWhenSystemProvided() throws Exception {
1781+
request.settings(Settings.builder().put(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID.getKey(), "private_setting").build());
1782+
request.settingsSystemProvided(true);
1783+
1784+
IndicesService indicesService = mock(IndicesService.class);
1785+
withTemporaryClusterService(((clusterService, threadPool) -> {
1786+
MetadataCreateIndexService service = new MetadataCreateIndexService(
1787+
Settings.EMPTY,
1788+
clusterService,
1789+
indicesService,
1790+
null,
1791+
createTestShardLimitService(randomIntBetween(1, 1000), clusterService),
1792+
newEnvironment(),
1793+
new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS),
1794+
threadPool,
1795+
null,
1796+
EmptySystemIndices.INSTANCE,
1797+
true,
1798+
IndexSettingProviders.EMPTY
1799+
);
1800+
1801+
try {
1802+
service.applyCreateIndexRequest(clusterService.state(), request, false, ActionListener.wrap(r -> {}, e -> {}));
1803+
} catch (Exception e) {
1804+
fail(e, "did not expect private setting to be rejected when system provided");
1805+
}
1806+
}));
1807+
1808+
ArgumentCaptor<IndexMetadata> captor = ArgumentCaptor.forClass(IndexMetadata.class);
1809+
verify(indicesService).withTempIndexService(captor.capture(), any());
1810+
IndexMetadata indexMetadata = captor.getValue();
1811+
assertThat(indexMetadata.getSettings().get(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID.getKey()), equalTo("private_setting"));
1812+
}
1813+
1814+
public void testIndexSettingProviderPrivateSetting() throws Exception {
1815+
IndicesService indicesService = mock(IndicesService.class);
1816+
withTemporaryClusterService(((clusterService, threadPool) -> {
1817+
MetadataCreateIndexService service = new MetadataCreateIndexService(
1818+
Settings.EMPTY,
1819+
clusterService,
1820+
indicesService,
1821+
null,
1822+
createTestShardLimitService(randomIntBetween(1, 1000), clusterService),
1823+
newEnvironment(),
1824+
new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS),
1825+
threadPool,
1826+
null,
1827+
EmptySystemIndices.INSTANCE,
1828+
true,
1829+
IndexSettingProviders.of(
1830+
(additionalSettings) -> additionalSettings.put(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME.getKey(), "private_setting")
1831+
)
1832+
);
1833+
1834+
try {
1835+
service.applyCreateIndexRequest(clusterService.state(), request, false, ActionListener.wrap(r -> {}, e -> {}));
1836+
} catch (Exception e) {
1837+
fail(e, "did not expect private setting to be rejected when added via IndexSettingProvider");
1838+
}
1839+
}));
1840+
1841+
ArgumentCaptor<IndexMetadata> captor = ArgumentCaptor.forClass(IndexMetadata.class);
1842+
verify(indicesService).withTempIndexService(captor.capture(), any());
1843+
IndexMetadata indexMetadata = captor.getValue();
1844+
assertThat(indexMetadata.getSettings().get(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME.getKey()), equalTo("private_setting"));
1845+
}
1846+
17421847
private IndexTemplateMetadata addMatchingTemplate(Consumer<IndexTemplateMetadata.Builder> configurator) {
17431848
IndexTemplateMetadata.Builder builder = templateMetadataBuilder("template1", "te*");
17441849
configurator.accept(builder);
@@ -1776,7 +1881,7 @@ private ShardLimitValidator randomShardLimitService() {
17761881

17771882
private void withTemporaryClusterService(BiConsumer<ClusterService, ThreadPool> consumer) {
17781883
final ThreadPool threadPool = new TestThreadPool(getTestName());
1779-
final ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool);
1884+
final ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool, projectId);
17801885
try {
17811886
consumer.accept(clusterService, threadPool);
17821887
} finally {

0 commit comments

Comments
 (0)