Support flattened fields as time series dimension fields (#95273)

Normally dimension fields are identified by means of a boolean parameter
at mapping time, time_series_dimension. Flattened fields do not have mappings,
other than identifying the top level field as a flattened field type. Moreover a boolean
is not enough to identify the top-level field as a dimension since we would like
users to be able to specify a subset of the fields in the flattened field to be dimensions
(not necessarily all of them). For this reason we introduce a new mapping parameter,
time_series_dimensions, which lists the fields, in any order, in the flattened field
that the user wants as dimensions. Field names must not include the root field name
and their name is the relative path from the root down to the leaf field name.

We require flattened fields to be indexed, to have doc values and disallow usage
of the ignore_above parameter together with time_series_dimensions.

Author: salvatore-campagna

docs/changelog/95273.yaml

@@ -0,0 +1,5 @@
+pr: 95273
+summary: Support flattened fields as time series dimension fields
+area: TSDB
+type: feature
+issues: []

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java

@@ -249,8 +249,9 @@ public void testInvalidTsdbTemplatesNoTimeSeriesDimensionAttribute() throws Exce
             assertThat(
                 e.getCause().getCause().getMessage(),
                 equalTo(
-                    "All fields that match routing_path must be keywords with [time_series_dimension: true] and "
-                        + "without the [script] parameter. [metricset] was not [time_series_dimension: true]."
+                    "All fields that match routing_path must be keywords with [time_series_dimension: true] "
+                        + "or flattened fields with a list of dimensions in [time_series_dimensions] and "
+                        + "without the [script] parameter. [metricset] was not a dimension."
                 )
             );
         }
@@ -316,7 +317,8 @@ public void testInvalidTsdbTemplatesNoKeywordFieldType() throws Exception {
         assertThat(
             e.getCause().getCause().getMessage(),
             equalTo(
-                "All fields that match routing_path must be keywords with [time_series_dimension: true] and "
+                "All fields that match routing_path must be keywords with [time_series_dimension: true] "
+                    + "or flattened fields with a list of dimensions in [time_series_dimensions] and "
                     + "without the [script] parameter. [metricset] was [long]."
             )
         );

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml

@@ -114,11 +114,11 @@ top level wildcard dim object:
 ---
 exact match object type:
   - skip:
-      version: " - 8.1.99"
-      reason: routing_path object type check improve in 8.2.0
+      version: " - 8.7.99"
+      reason: routing_path error message updated in 8.8.0
 
   - do:
-      catch: '/All fields that match routing_path must be keywords with \[time_series_dimension: true\] and without the \[script\] parameter. \[dim\] was \[object\]./'
+      catch: '/All fields that match routing_path must be keywords with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[dim\] was \[object\]./'
       indices.create:
         index: tsdb_index
         body:
@@ -147,11 +147,11 @@ exact match object type:
 ---
 non keyword matches routing_path:
   - skip:
-      version: " - 8.0.99"
-      reason: introduced in 8.1.0
+      version: " - 8.7.99"
+      reason: routing_path error message updated in 8.8.0
 
   - do:
-      catch: '/All fields that match routing_path must be keywords with \[time_series_dimension: true\] and without the \[script\] parameter. \[@timestamp\] was \[date\]./'
+      catch: '/All fields that match routing_path must be keywords with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[@timestamp\] was \[date\]./'
       indices.create:
           index: test_index
           body:
@@ -234,8 +234,8 @@ runtime field matching routing path:
 ---
 "dynamic: runtime matches routing_path":
   - skip:
-      version: " - 8.1.99"
-      reason: tsdb indexing changed in 8.2.0
+      version: " - 8.7.99"
+      reason: routing_path error message updated in 8.8.0
 
   - do:
       indices.create:
@@ -266,7 +266,7 @@ runtime field matching routing path:
         body:
           - '{"index": {}}'
           - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim_kw": "dim", "dim": {"foo": "a"}}'
-  - match: {items.0.index.error.reason: "All fields that match routing_path must be keywords with [time_series_dimension: true] and without the [script] parameter. [dim.foo] was a runtime [keyword]."}
+  - match: {items.0.index.error.reason: "All fields that match routing_path must be keywords with [time_series_dimension: true] or flattened fields with a list of dimensions in [time_series_dimensions] and without the [script] parameter. [dim.foo] was a runtime [keyword]."}
 
 ---
 "dynamic: false matches routing_path":

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml

@@ -204,9 +204,11 @@ protected final LeafFactory leafFactory(SearchExecutionContext context) {
     }
 
     @Override
-    public void validateMatchedRoutingPath() {
+    public void validateMatchedRoutingPath(final String routingPath) {
         throw new IllegalArgumentException(
-            "All fields that match routing_path must be keywords with [time_series_dimension: true] "
+            "All fields that match routing_path "
+                + "must be keywords with [time_series_dimension: true] "
+                + "or flattened fields with a list of dimensions in [time_series_dimensions] "
                 + "and without the [script] parameter. ["
                 + name()
                 + "] was a runtime ["

server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java

@@ -105,13 +105,14 @@ public void validate(IndexSettings settings, boolean checkLimits) {
         List<String> routingPaths = settings.getIndexMetadata().getRoutingPaths();
         for (String path : routingPaths) {
             for (String match : mappingLookup.getMatchingFieldNames(path)) {
-                mappingLookup.getFieldType(match).validateMatchedRoutingPath();
+                mappingLookup.getFieldType(match).validateMatchedRoutingPath(path);
             }
             for (String objectName : mappingLookup.objectMappers().keySet()) {
                 // object type is not allowed in the routing paths
                 if (path.equals(objectName)) {
                     throw new IllegalArgumentException(
                         "All fields that match routing_path must be keywords with [time_series_dimension: true] "
+                            + "or flattened fields with a list of dimensions in [time_series_dimensions] "
                             + "and without the [script] parameter. ["
                             + objectName
                             + "] was [object]."

server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java

@@ -779,19 +779,22 @@ public boolean isDimension() {
         }
 
         @Override
-        public void validateMatchedRoutingPath() {
+        public void validateMatchedRoutingPath(final String routingPath) {
             if (false == isDimension) {
                 throw new IllegalArgumentException(
-                    "All fields that match routing_path must be keywords with [time_series_dimension: true] "
-                        + "and without the [script] parameter. ["
+                    "All fields that match routing_path "
+                        + "must be keywords with [time_series_dimension: true] "
+                        + "or flattened fields with a list of dimensions in [time_series_dimensions] and "
+                        + "without the [script] parameter. ["
                         + name()
-                        + "] was not [time_series_dimension: true]."
+                        + "] was not a dimension."
                 );
             }
             if (scriptValues != null) {
                 throw new IllegalArgumentException(
                     "All fields that match routing_path must be keywords with [time_series_dimension: true] "
-                        + "and without the [script] parameter. ["
+                        + "or flattened fields with a list of dimensions in [time_series_dimensions] and "
+                        + "without the [script] parameter. ["
                         + name()
                         + "] has a [script] parameter."
                 );

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -46,6 +46,8 @@
 import java.io.IOException;
 import java.time.ZoneId;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.function.Function;
@@ -188,6 +190,15 @@ public boolean isDimension() {
         return false;
     }
 
+    /**
+     * @return a list of dimension fields. Expected to be used by fields that have
+     * nested fields or that, in some way, identify a collection of fields by means
+     * of a top level field (like flattened fields).
+     */
+    public List<String> dimensions() {
+        return Collections.emptyList();
+    }
+
     /**
      * @return metric type or null if the field is not a metric field
      */
@@ -604,10 +615,12 @@ public TermsEnum getTerms(IndexReader reader, String prefix, boolean caseInsensi
     /**
      * Validate that this field can be the target of {@link IndexMetadata#INDEX_ROUTING_PATH}.
      */
-    public void validateMatchedRoutingPath() {
+    public void validateMatchedRoutingPath(String routingPath) {
         throw new IllegalArgumentException(
-            "All fields that match routing_path must be keywords with [time_series_dimension: true] "
-                + "and without the [script] parameter. ["
+            "All fields that match routing_path "
+                + "must be keywords with [time_series_dimension: true] "
+                + "or flattened fields with a list of dimensions in [time_series_dimensions] and "
+                + "without the [script] parameter. ["
                 + name()
                 + "] was ["
                 + typeName()