Store dimensions in custom metadata rather than an index setting

Author: felixbarny

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java

@@ -48,7 +48,6 @@
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.ReindexAction;
 import org.elasticsearch.index.reindex.ReindexRequest;
-import org.elasticsearch.indices.InvalidIndexTemplateException;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.reindex.ReindexPlugin;
 import org.elasticsearch.rest.RestStatus;
@@ -69,7 +68,6 @@
 import static org.elasticsearch.test.MapMatcher.matchesMap;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
-import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -302,10 +300,10 @@ public void testInvalidTsdbTemplatesNoTimeSeriesDimensionAttribute() throws Exce
                     .build()
             );
             var e = expectThrows(
-                InvalidIndexTemplateException.class,
+                IllegalArgumentException.class,
                 () -> client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet()
             );
-            assertThat(e.getMessage(), containsString("[index.mode=time_series] requires a non-empty [index.routing_path]"));
+            assertThat(e.getCause().getMessage(), containsString("[index.mode=time_series] requires a non-empty [index.routing_path]"));
         }
     }
 
@@ -649,7 +647,7 @@ public void testAddDimensionToMapping() throws Exception {
         );
         assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDsRequest));
 
-        assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), equalTo(List.of("metricset")));
+        assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of()));
 
         // put mapping with k8s.pod.uid as another time series dimension
         var putMappingRequest = new PutMappingRequest(dataStreamName).source("""
@@ -664,7 +662,7 @@ public void testAddDimensionToMapping() throws Exception {
             """, XContentType.JSON);
         assertAcked(client().execute(TransportPutMappingAction.TYPE, putMappingRequest).actionGet());
 
-        assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), containsInAnyOrder("metricset", "k8s.pod.name"));
+        assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of()));
 
         indexWithPodNames(dataStreamName, Instant.now(), Map.of(), "dog", "cat");
     }

modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java

@@ -11,7 +11,9 @@
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
+import org.elasticsearch.cluster.routing.TimeSeriesDimensionsMetadataAccess;
 import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
@@ -36,8 +38,8 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 
-import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_DIMENSIONS;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH;
 
 /**
@@ -63,7 +65,8 @@ public Settings getAdditionalIndexSettings(
         ProjectMetadata projectMetadata,
         Instant resolvedAt,
         Settings indexTemplateAndCreateRequestSettings,
-        List<CompressedXContent> combinedTemplateMappings
+        List<CompressedXContent> combinedTemplateMappings,
+        ImmutableOpenMap.Builder<String, Map<String, String>> extraCustomMetadata
     ) {
         if (dataStreamName != null) {
             DataStream dataStream = projectMetadata.dataStreams().get(dataStreamName);
@@ -127,7 +130,11 @@ public Settings getAdditionalIndexSettings(
                             dimensions
                         );
                         if (dimensions.isEmpty() == false) {
-                            builder.putList(matchesAllDimensions ? INDEX_DIMENSIONS.getKey() : INDEX_ROUTING_PATH.getKey(), dimensions);
+                            if (matchesAllDimensions) {
+                                TimeSeriesDimensionsMetadataAccess.addToCustomMetadata(extraCustomMetadata, dimensions);
+                            } else {
+                                builder.putList(INDEX_ROUTING_PATH.getKey(), dimensions);
+                            }
                         }
                     }
                     return builder.build();
@@ -139,15 +146,19 @@ public Settings getAdditionalIndexSettings(
     }
 
     /**
-     * This is called when mappings are updated, so that the {@link IndexMetadata#INDEX_DIMENSIONS}
+     * This is called when mappings are updated, so that the {@link IndexMetadata#getTimeSeriesDimensions()}
      * and {@link IndexMetadata#INDEX_ROUTING_PATH} settings are updated to match the new mappings.
-     * Updates {@link IndexMetadata#INDEX_DIMENSIONS} if a new dimension field is added to the mappings,
+     * Updates {@link IndexMetadata#getTimeSeriesDimensions} if a new dimension field is added to the mappings,
      * or sets {@link IndexMetadata#INDEX_ROUTING_PATH} if a new dimension field is added that doesn't allow for matching all
      * dimension fields via a wildcard pattern.
      */
     @Override
-    public Settings onUpdateMappings(IndexMetadata indexMetadata, DocumentMapper documentMapper) {
-        List<String> indexDimensions = INDEX_DIMENSIONS.get(indexMetadata.getSettings());
+    public Settings onUpdateMappings(
+        IndexMetadata indexMetadata,
+        ImmutableOpenMap.Builder<String, Map<String, String>> extraCustomMetadata,
+        DocumentMapper documentMapper
+    ) {
+        List<String> indexDimensions = indexMetadata.getTimeSeriesDimensions();
         if (indexDimensions.isEmpty()) {
             return Settings.EMPTY;
         }
@@ -158,8 +169,10 @@ public Settings onUpdateMappings(IndexMetadata indexMetadata, DocumentMapper doc
             return Settings.EMPTY;
         }
         if (matchesAllDimensions) {
-            return Settings.builder().putList(INDEX_DIMENSIONS.getKey(), newIndexDimensions).build();
+            TimeSeriesDimensionsMetadataAccess.addToCustomMetadata(extraCustomMetadata, newIndexDimensions);
+            return Settings.EMPTY;
         } else {
+            // If the new dimensions don't match all potential dimension fields, we need to set index.routing_path
             return Settings.builder().putList(INDEX_ROUTING_PATH.getKey(), newIndexDimensions).build();
         }
     }

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java

@@ -36,6 +36,7 @@
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.project.ProjectResolver;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Nullable;
@@ -205,8 +206,8 @@ static IndexMode resolveMode(
                 state.metadata(),
                 Instant.now(),
                 settings,
-                List.of()
-            );
+                List.of(),
+                ImmutableOpenMap.builder());
             var rawMode = addlSettinsg.get(IndexSettings.MODE.getKey());
             if (rawMode != null) {
                 indexMode = Enum.valueOf(IndexMode.class, rawMode.toUpperCase(Locale.ROOT));