Add ability to provide extra custom index metadata

Author: felixbarny

modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
@@ -37,6 +38,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH;
 
@@ -63,7 +65,8 @@ public Settings getAdditionalIndexSettings(
         ProjectMetadata projectMetadata,
         Instant resolvedAt,
         Settings indexTemplateAndCreateRequestSettings,
-        List<CompressedXContent> combinedTemplateMappings
+        List<CompressedXContent> combinedTemplateMappings,
+        ImmutableOpenMap.Builder<String, Map<String, String>> extraCustomMetadata
     ) {
         if (dataStreamName != null) {
             DataStream dataStream = projectMetadata.dataStreams().get(dataStreamName);

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java

@@ -36,6 +36,7 @@
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.project.ProjectResolver;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Nullable;
@@ -205,7 +206,8 @@ static IndexMode resolveMode(
                 state.metadata(),
                 Instant.now(),
                 settings,
-                List.of()
+                List.of(),
+                ImmutableOpenMap.builder()
             );
             var rawMode = addlSettinsg.get(IndexSettings.MODE.getKey());
             if (rawMode != null) {

modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Strings;
@@ -82,7 +83,8 @@ public void testGetAdditionalIndexSettings() throws Exception {
             projectMetadata,
             now,
             settings,
-            List.of(new CompressedXContent(mapping))
+            List.of(new CompressedXContent(mapping)),
+            ImmutableOpenMap.builder()
         );
         // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting:
         // (in production the index.mode setting is usually provided in an index or component template)
@@ -105,7 +107,7 @@ public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throw
                 "_doc": {
                     "properties": {
                         "field1": {
-                            "type": "keyword"
+                            "type": "keyword",
                             "time_series_dimension": true
                         },
                         "field2": {
@@ -127,7 +129,8 @@ public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throw
             projectMetadata,
             now,
             settings,
-            List.of(new CompressedXContent(mapping))
+            List.of(new CompressedXContent(mapping)),
+            ImmutableOpenMap.builder()
         );
         // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting:
         // (in production the index.mode setting is usually provided in an index or component template)
@@ -197,7 +200,8 @@ public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception {
             projectMetadata,
             now,
             settings,
-            List.of(new CompressedXContent(mapping1), new CompressedXContent(mapping2), new CompressedXContent(mapping3))
+            List.of(new CompressedXContent(mapping1), new CompressedXContent(mapping2), new CompressedXContent(mapping3)),
+            ImmutableOpenMap.builder()
         );
         // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting:
         // (in production the index.mode setting is usually provided in an index or component template)
@@ -222,7 +226,8 @@ public void testGetAdditionalIndexSettingsNoMappings() {
             projectMetadata,
             now,
             settings,
-            List.of()
+            List.of(),
+            ImmutableOpenMap.builder()
         );
         // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting:
         // (in production the index.mode setting is usually provided in an index or component template)
@@ -247,7 +252,8 @@ public void testGetAdditionalIndexSettingsLookAheadTime() throws Exception {
             projectMetadata,
             now,
             settings,
-            List.of(new CompressedXContent("{}"))
+            List.of(new CompressedXContent("{}")),
+            ImmutableOpenMap.builder()
         );
         // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting:
         // (in production the index.mode setting is usually provided in an index or component template)
@@ -272,7 +278,8 @@ public void testGetAdditionalIndexSettingsLookBackTime() throws Exception {
             projectMetadata,
             now,
             settings,
-            List.of(new CompressedXContent("{}"))
+            List.of(new CompressedXContent("{}")),
+            ImmutableOpenMap.builder()
         );
         // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting:
         // (in production the index.mode setting is usually provided in an index or component template)
@@ -304,7 +311,8 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreated() throws Exce
             projectMetadata,
             now,
             settings,
-            List.of(new CompressedXContent("{}"))
+            List.of(new CompressedXContent("{}")),
+            ImmutableOpenMap.builder()
         );
         assertThat(result.size(), equalTo(2));
         assertThat(result.get(IndexSettings.TIME_SERIES_START_TIME.getKey()), equalTo(FORMATTER.format(currentEnd)));
@@ -341,7 +349,8 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreatedTimeSettingsMi
                 projectMetadata,
                 now,
                 settings,
-                null
+                null,
+                ImmutableOpenMap.builder()
             )
         );
         assertThat(
@@ -367,7 +376,8 @@ public void testGetAdditionalIndexSettingsNonTsdbTemplate() {
             projectMetadata,
             Instant.ofEpochMilli(1L),
             settings,
-            null
+            null,
+            ImmutableOpenMap.builder()
         );
         assertThat(result.size(), equalTo(0));
     }
@@ -389,7 +399,8 @@ public void testGetAdditionalIndexSettingsMigrateToTsdb() {
             projectMetadata,
             now,
             settings,
-            List.of()
+            List.of(),
+            ImmutableOpenMap.builder()
         );
         // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting:
         // (in production the index.mode setting is usually provided in an index or component template)
@@ -419,7 +430,8 @@ public void testGetAdditionalIndexSettingsDowngradeFromTsdb() {
             projectMetadata,
             Instant.ofEpochMilli(1L),
             settings,
-            List.of()
+            List.of(),
+            ImmutableOpenMap.builder()
         );
         assertThat(result.size(), equalTo(0));
     }
@@ -698,7 +710,8 @@ private Settings generateTsdbSettings(String mapping, Instant now) throws IOExce
             projectMetadata,
             now,
             settings,
-            List.of(new CompressedXContent(mapping))
+            List.of(new CompressedXContent(mapping)),
+            ImmutableOpenMap.builder()
         );
         // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting:
         // (in production the index.mode setting is usually provided in an index or component template)

modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java

@@ -22,13 +22,16 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.metadata.Template;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.IndexSettingProvider;
 import org.elasticsearch.index.IndexSettingProviders;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.indices.SystemIndices;
@@ -490,18 +493,21 @@ public void testProvidersAffectMode() {
             ClusterSettings.createBuiltInClusterSettings(),
             dataStreamGlobalRetentionSettings,
             emptyDataStreamFailureStoreSettings,
-            new IndexSettingProviders(
-                Set.of(
-                    (
-                        indexName,
-                        dataStreamName,
-                        templateIndexMode,
-                        metadata,
-                        resolvedAt,
-                        indexTemplateAndCreateRequestSettings,
-                        combinedTemplateMappings) -> Settings.builder().put("index.mode", IndexMode.LOOKUP).build()
-                )
-            ),
+            new IndexSettingProviders(Set.of(new IndexSettingProvider() {
+                @Override
+                public Settings getAdditionalIndexSettings(
+                    String indexName,
+                    String dataStreamName,
+                    IndexMode templateIndexMode,
+                    ProjectMetadata metadata,
+                    Instant resolvedAt,
+                    Settings indexTemplateAndCreateRequestSettings,
+                    List<CompressedXContent> combinedTemplateMappings,
+                    ImmutableOpenMap.Builder<String, Map<String, String>> extraCustomMetadata
+                ) {
+                    return Settings.builder().put("index.mode", IndexMode.LOOKUP).build();
+                }
+            })),
             null
         );
         assertThat(response.getDataStreams().getFirst().getIndexModeName(), equalTo("lookup"));

server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.cluster.project.ProjectResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -285,7 +286,8 @@ public static Template resolveTemplate(
                 simulatedProject,
                 now,
                 templateSettings,
-                mappings
+                mappings,
+                ImmutableOpenMap.builder()
             );
             MetadataCreateIndexService.validateAdditionalSettings(provider, result, additionalSettings);
             dummySettings.put(result);

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java

@@ -2133,6 +2133,11 @@ public Builder removeAllAliases() {
             return this;
         }
 
+        public Builder putCustom(Map<String, ? extends Map<String, String>> customMetadata) {
+            customMetadata.forEach(this::putCustom);
+            return this;
+        }
+
         public Builder putCustom(String type, Map<String, String> customIndexMetadata) {
             this.customMetadata.put(type, new DiffableStringMap(customIndexMetadata));
             return this;