OTel mappings: avoid metrics to be rejected when attributes are malformed (#114856)

Author: felixbarny

docs/changelog/114856.yaml

@@ -0,0 +1,5 @@
+pr: 114856
+summary: "OTel mappings: avoid metrics to be rejected when attributes are malformed"
+area: Data streams
+type: bug
+issues: []

x-pack/plugin/otel-data/src/main/resources/component-templates/[email protected]

@@ -3,6 +3,8 @@ _meta:
   description: Default mappings for the OpenTelemetry metrics index template installed by x-pack
   managed: true
 template:
+  settings:
+    index.mapping.ignore_malformed: true
   mappings:
     properties:
       start_timestamp:
@@ -19,27 +21,22 @@ template:
       - histogram:
           mapping:
             type: histogram
-            ignore_malformed: true
       - counter_long:
           mapping:
             type: long
             time_series_metric: counter
-            ignore_malformed: true
       - gauge_long:
           mapping:
             type: long
             time_series_metric: gauge
-            ignore_malformed: true
       - counter_double:
           mapping:
             type: double
             time_series_metric: counter
-            ignore_malformed: true
       - gauge_double:
           mapping:
             type: double
             time_series_metric: gauge
-            ignore_malformed: true
       - summary:
           mapping:
             type: aggregate_metric_double

x-pack/plugin/otel-data/src/main/resources/index-templates/[email protected]

@@ -27,14 +27,3 @@ template:
       data_stream.type:
         type: constant_keyword
         value: metrics
-    dynamic_templates:
-      - ecs_ip:
-          mapping:
-            type: ip
-          path_match: [ "ip", "*.ip", "*_ip" ]
-          match_mapping_type: string
-      - all_strings_to_keywords:
-          mapping:
-            ignore_above: 1024
-            type: keyword
-          match_mapping_type: string

x-pack/plugin/otel-data/src/main/resources/resources.yaml

@@ -1,7 +1,7 @@
 # "version" holds the version of the templates and ingest pipelines installed
 # by xpack-plugin otel-data. This must be increased whenever an existing template is
 # changed, in order for it to be updated on Elasticsearch upgrade.
-version: 5
+version: 6
 
 component-templates:
   - otel@mappings

x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml

@@ -245,3 +245,37 @@ IP dimensions:
   - match: { .$idx0name.mappings.properties.metrics.properties.summary.type: 'aggregate_metric_double' }
   - match: { .$idx0name.mappings.properties.metrics.properties.summary_minmax.type: 'aggregate_metric_double' }
   - match: { .$idx0name.mappings.properties.metrics.properties.histogram.type: 'histogram' }
+---
+Empty IP field:
+  - do:
+      bulk:
+        index: metrics-generic.otel-default
+        refresh: true
+        body:
+          - create: {"dynamic_templates":{"metrics.foo.bar":"counter_long"}}
+          - "@timestamp": 2024-07-18T14:48:33.467654000Z
+            resource:
+              attributes:
+                host.name: localhost
+                host.ip: ""
+            metrics:
+              foo.bar: 42
+  - is_false: errors
+
+  - do:
+      indices.get_data_stream:
+        name: metrics-generic.otel-default
+  - set: { data_streams.0.indices.0.index_name: idx0name }
+
+  - do:
+      indices.get_mapping:
+        index: $idx0name
+        expand_wildcards: hidden
+  - match: { .$idx0name.mappings.properties.resource.properties.attributes.properties.host\.ip.type: 'ip' }
+  - do:
+      search:
+        index: metrics-generic.otel-default
+        body:
+          fields: ["*"]
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._ignored: ["resource.attributes.host.ip"] }