[9.2] Fix multi fields in downsampling (#138869) (#139049)

* Fix multi fields in downsampling (#138869)

(cherry picked from commit 5918605)

Author: gmarouli

docs/changelog/138869.yaml

@@ -0,0 +1,5 @@
+pr: 138869
+summary: Fix multi fields in downsampling
+area: Downsampling
+type: bug
+issues: []

server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingVisitor.java

@@ -17,13 +17,22 @@ public final class MappingVisitor {
     private MappingVisitor() {}
 
     public static void visitMapping(Map<String, ?> mapping, BiConsumer<String, Map<String, ?>> fieldMappingConsumer) {
-        visitMapping(mapping, "", fieldMappingConsumer);
+        visitMapping(mapping, "", fieldMappingConsumer, fieldMappingConsumer);
+    }
+
+    public static void visitMapping(
+        Map<String, ?> mapping,
+        BiConsumer<String, Map<String, ?>> fieldMappingConsumer,
+        final BiConsumer<String, Map<String, ?>> multiFieldsMappingConsumer
+    ) {
+        visitMapping(mapping, "", fieldMappingConsumer, multiFieldsMappingConsumer);
     }
 
     private static void visitMapping(
         final Map<String, ?> mapping,
         final String path,
-        final BiConsumer<String, Map<String, ?>> fieldMappingConsumer
+        final BiConsumer<String, Map<String, ?>> propertiesMappingConsumer,
+        final BiConsumer<String, Map<String, ?>> multiFieldsMappingConsumer
     ) {
         Object properties = mapping.get("properties");
         if (properties instanceof Map) {
@@ -36,8 +45,8 @@ private static void visitMapping(
                     @SuppressWarnings("unchecked")
                     Map<String, ?> fieldMapping = (Map<String, ?>) v;
                     final String prefix = path + entry.getKey();
-                    fieldMappingConsumer.accept(prefix, fieldMapping);
-                    visitMapping(fieldMapping, prefix + ".", fieldMappingConsumer);
+                    propertiesMappingConsumer.accept(prefix, fieldMapping);
+                    visitMapping(fieldMapping, prefix + ".", propertiesMappingConsumer, multiFieldsMappingConsumer);
 
                     // Multi fields
                     Object fieldsO = fieldMapping.get("fields");
@@ -49,7 +58,7 @@ private static void visitMapping(
                             if (v2 instanceof Map) {
                                 @SuppressWarnings("unchecked")
                                 Map<String, ?> fieldMapping2 = (Map<String, ?>) v2;
-                                fieldMappingConsumer.accept(prefix + "." + subfieldEntry.getKey(), fieldMapping2);
+                                multiFieldsMappingConsumer.accept(prefix + "." + subfieldEntry.getKey(), fieldMapping2);
                             }
                         }
                     }

server/src/main/resources/transport/definitions/referable/downsample_add_multi_field_sources.csv

@@ -0,0 +1 @@
+9232000,9185013,9112016,8841077

server/src/main/resources/transport/upper_bounds/8.19.csv

@@ -1 +1 @@
-initial_8.19.9,8841076
+downsample_add_multi_field_sources,8841077

server/src/main/resources/transport/upper_bounds/9.1.csv

@@ -1 +1 @@
-initial_9.1.9,9112015
+downsample_add_multi_field_sources,9112016

server/src/main/resources/transport/upper_bounds/9.2.csv

@@ -1 +1 @@
-initial_9.2.3,9185012
+downsample_add_multi_field_sources,9185013

server/src/main/resources/transport/upper_bounds/9.3.csv

@@ -1 +1 @@
-esql_use_minimum_version_for_enrich_resolution,9231000
+downsample_add_multi_field_sources,9232000

server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingVisitorTests.java

@@ -30,6 +30,10 @@ private static void collectTypes(Map<String, ?> mapping, Set<String> types) {
         });
     }
 
+    private static void collectFieldsAndSubFields(Map<String, ?> mapping, Set<String> fields, Set<String> subFields) {
+        MappingVisitor.visitMapping(mapping, (f, m) -> fields.add(f), (f, m) -> subFields.add(f));
+    }
+
     public void testCountTopLevelFields() {
         Map<String, Object> mapping = new HashMap<>();
         Set<String> fields = new HashSet<>();
@@ -45,14 +49,14 @@ public void testCountTopLevelFields() {
         collectTypes(mapping, fields);
         assertEquals(Collections.singleton("keyword"), fields);
 
-        Map<String, Object> IndexField = new HashMap<>();
-        IndexField.put("type", "integer");
-        properties.put("bar", IndexField);
+        Map<String, Object> indexField = new HashMap<>();
+        indexField.put("type", "integer");
+        properties.put("bar", indexField);
         fields = new HashSet<>();
         collectTypes(mapping, fields);
         assertEquals(new HashSet<>(Arrays.asList("keyword", "integer")), fields);
 
-        properties.put("baz", IndexField);
+        properties.put("baz", indexField);
         fields = new HashSet<>();
         collectTypes(mapping, fields);
         assertEquals(new HashSet<>(Arrays.asList("keyword", "integer")), fields);
@@ -80,6 +84,35 @@ public void testCountMultiFields() {
         assertEquals(new HashSet<>(Arrays.asList("keyword", "text")), usedFields);
     }
 
+    public void testFieldsAndMultiFields() {
+        Map<String, Object> keywordType = new HashMap<>();
+        keywordType.put("type", "keyword");
+
+        Map<String, Object> textType = new HashMap<>();
+        textType.put("type", "text");
+
+        Map<String, Object> multiFields = new HashMap<>();
+        multiFields.put("keyword", keywordType);
+        textType.put("fields", multiFields);
+
+        Map<String, Object> subObject = new HashMap<>();
+        subObject.put("properties", Map.of("baz", keywordType));
+        subObject.put("type", "keyword");
+
+        Map<String, Object> properties = new HashMap<>();
+        properties.put("foo", textType);
+        properties.put("bar", subObject);
+
+        Map<String, Object> mapping = new HashMap<>();
+        mapping.put("properties", properties);
+
+        Set<String> fields = new HashSet<>();
+        Set<String> subFields = new HashSet<>();
+        collectFieldsAndSubFields(mapping, fields, subFields);
+        assertEquals(Set.of("foo", "bar", "bar.baz"), fields);
+        assertEquals(Set.of("foo.keyword"), subFields);
+    }
+
     public void testCountInnerFields() {
         Map<String, Object> keywordField = new HashMap<>();
         keywordField.put("type", "keyword");

x-pack/plugin/downsample/qa/rest/build.gradle

@@ -25,7 +25,7 @@ dependencies {
 
 restResources {
   restApi {
-    include '_common', 'bulk', 'cluster', 'indices', 'search', 'ingest.put_pipeline', 'ingest.delete_pipeline'
+    include '_common', 'bulk', 'cluster', 'indices', 'search', 'ingest.put_pipeline', 'ingest.delete_pipeline', 'capabilities'
   }
 }
 

x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/90_multi_fields.yml

@@ -0,0 +1,155 @@
+setup:
+  - requires:
+      capabilities:
+        - method: POST
+          path: /{index}/_downsample/{target_index}
+          capabilities: ['downsample.multi_field_fix']
+      test_runner_features: [ "capabilities" ]
+      reason: Requires the fix for downsampling multi fields
+
+---
+"Downsample index with multi-fields":
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 1
+            index:
+              mode: time_series
+              routing_path: [ dimension ]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              dimension:
+                type: keyword
+                time_series_dimension: true
+              label-dimension:
+                type: keyword
+                fields:
+                  dimension:
+                    type: keyword
+                    time_series_dimension: true
+              label-text:
+                type: text
+                fields:
+                  keyword:
+                    type: keyword
+                  other-keyword:
+                    type: keyword
+              metric:
+                type: long
+                time_series_metric: gauge
+  - is_true: shards_acknowledged
+
+  - do:
+      bulk:
+        refresh: true
+        index: test
+        body:
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dimension": "pod", "label-dimension": "my label", "label-text": "my text", "metric": 1}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:24.467Z", "dimension": "pod", "label-dimension": "my label", "label-text": "my other text", "metric": 3}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T20:50:44.467Z", "dimension": "pod", "label-dimension": "my label", "label-text": "my random text", "metric": 2}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T20:51:04.467Z", "dimension": "pod", "label-dimension": "my other label", "label-text": "some text", "metric": 4}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:03.142Z", "dimension": "pod", "label-dimension": "my label", "label-text": "some other text", "metric": 6}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:23.142Z", "dimension": "pod", "label-dimension": "my label", "label-text": "some random text", "metric": 3}'
+
+  - do:
+      indices.put_settings:
+        index: test
+        body:
+          index.blocks.write: true
+  - is_true: acknowledged
+
+  - do:
+      indices.downsample:
+        index: test
+        target_index: test-downsample
+        body: >
+          {
+            "fixed_interval": "1h"
+          }
+  - is_true: acknowledged
+
+  - do:
+      search:
+        index: test-downsample
+        body:
+          sort: [ "@timestamp", "_tsid" ]
+
+  - length: { hits.hits: 3 }
+  - match: { hits.hits.0._source._doc_count: 4 }
+  - match: { hits.hits.0._source.dimension: pod }
+  - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
+  - match: { hits.hits.0._source.label-dimension: "my label" }
+  - match: { hits.hits.0._source.label-text: "my other text" }
+  - match: { hits.hits.0._source.metric.sum: 13 }
+  - match: { hits.hits.0._source.metric.value_count: 4 }
+  - match: { hits.hits.0._source.metric.min: 1 }
+  - match: { hits.hits.0._source.metric.max: 6 }
+
+---
+"Downsampling index fails with metrics as multi-fields":
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 1
+            index:
+              mode: time_series
+              routing_path: [ dimension ]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              dimension:
+                type: keyword
+                time_series_dimension: true
+              cpu_usage:
+                type: long
+                fields:
+                  gauge:
+                    type: long
+                    time_series_metric: gauge
+  - is_true: shards_acknowledged
+
+  - do:
+      bulk:
+        refresh: true
+        index: test
+        body:
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dimension": "pod", "cpu_usage": 1}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:24.467Z", "dimension": "pod", "cpu_usage": 3}'
+
+  - do:
+      indices.put_settings:
+        index: test
+        body:
+          index.blocks.write: true
+  - is_true: acknowledged
+
+  - do:
+      catch: /Downsampling failed because index mapping contains time series metrics as a multi field/
+      indices.downsample:
+        index: test
+        target_index: test-downsample
+        body: >
+          {
+            "fixed_interval": "1h"
+          }