Merge remote-tracking branch 'origin/main' into lucene_snapshot

Author: elasticsearchmachine

x-pack/plugin/otel-data/src/main/resources/component-templates/[email protected]

@@ -14,8 +14,7 @@ template:
       attributes:
         type: passthrough
         dynamic: true
-        priority: 10
-        time_series_dimension: true
+        priority: 20
         properties:
           exception.type:
             type: keyword
@@ -40,13 +39,28 @@ template:
       log.level:
         type: alias
         path: severity_text
-      body_text:
-        type: match_only_text
+      body:
+        type: object
+        properties:
+          text:
+            type: match_only_text
+          flattened:
+            # this is used for complex bodies of regular log records
+            # using the flattened field type avoids mapping issues which can be caused by logs containing arbitrary JSON objects
+            # the tradeoff is that the flattened field type is currently not supported well by Kibana and has other limitations
+            type: flattened
+          structured:
+            # this is used for events
+            # events are also represented as log records
+            # the event.name attribute uniquely identifies event structure / type of the payload (body)
+            # see also https://github.com/open-telemetry/semantic-conventions/blob/main/docs/general/events.md
+            # this makes them less prone to mapping issues, which is why we're enabling dynamic mappings
+            type: passthrough
+            dynamic: true
+            priority: 10
       message:
         type: alias
-        path: body_text
-      body_structured:
-        type: flattened
+        path: body.text
       trace_id:
         type: keyword
       trace.id:

x-pack/plugin/otel-data/src/main/resources/component-templates/[email protected]

@@ -10,7 +10,7 @@ template:
       metrics:
         type: passthrough
         dynamic: true
-        priority: 1
+        priority: 10
       unit:
         type: keyword
         time_series_dimension: true

x-pack/plugin/otel-data/src/main/resources/component-templates/[email protected]

@@ -20,7 +20,7 @@ template:
       attributes:
         type: passthrough
         dynamic: true
-        priority: 10
+        priority: 20
         time_series_dimension: true
       dropped_attributes_count:
         type: long
@@ -39,7 +39,7 @@ template:
           attributes:
             type: passthrough
             dynamic: true
-            priority: 20
+            priority: 30
             time_series_dimension: true
       resource:
         properties:
@@ -51,7 +51,7 @@ template:
           attributes:
             type: passthrough
             dynamic: true
-            priority: 30
+            priority: 40
             time_series_dimension: true
     dynamic_templates:
       - complex_attributes:

x-pack/plugin/otel-data/src/main/resources/component-templates/[email protected]

@@ -11,7 +11,7 @@ template:
           attributes:
             type: passthrough
             dynamic: true
-            priority: 30
+            priority: 40
             time_series_dimension: true
             properties:
               host.name:

x-pack/plugin/otel-data/src/main/resources/resources.yaml

@@ -1,7 +1,7 @@
 # "version" holds the version of the templates and ingest pipelines installed
 # by xpack-plugin otel-data. This must be increased whenever an existing template is
 # changed, in order for it to be updated on Elasticsearch upgrade.
-version: 4
+version: 5
 
 component-templates:
   - otel@mappings

x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml

@@ -11,7 +11,7 @@ setup:
         refresh: true
         body:
           - create: {}
-          - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "attributes": { "foo": "bar"}, "body_text":"Error: Unable to connect to the database.","severity_text":"ERROR","severity_number":3,"trace_id":"abc123xyz456def789ghi012jkl345"}'
+          - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "attributes": { "foo": "bar"}, "body":{"text":"Error: Unable to connect to the database."},"severity_text":"ERROR","severity_number":3,"trace_id":"abc123xyz456def789ghi012jkl345"}'
   - is_false: errors
   - do:
       search:
@@ -39,7 +39,8 @@ setup:
             attributes:
               foo: [3, 2, 1]
               bar: [b, c, a]
-            body_text: "Error: Unable to connect to the database."
+            body:
+              text: "Error: Unable to connect to the database."
             severity_text: ERROR
   - is_false: errors
   - do:
@@ -78,7 +79,7 @@ setup:
         refresh: true
         body:
           - create: {}
-          - '{"@timestamp":"2024-07-18T14:49:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "body_text":"error1"}'
+          - '{"@timestamp":"2024-07-18T14:49:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "body": {"text":"error1"}}'
   - is_false: errors
   - do:
       indices.get_data_stream:
@@ -90,3 +91,57 @@ setup:
   - is_true: $datastream-backing-index
   - match: { .$datastream-backing-index.settings.index.sort.field.0: "resource.attributes.host.name" }
   - match: { .$datastream-backing-index.settings.index.sort.field.1: "@timestamp" }
+---
+Event body:
+  - do:
+      bulk:
+        index: logs-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - "@timestamp": 2024-07-18T14:48:33.467654000Z
+            resource:
+              attributes:
+                service.name: my-service
+            attributes:
+              event.name: foo
+            body:
+              structured:
+                foo:
+                  bar: baz
+  - is_false: errors
+  - do:
+      indices.get_data_stream:
+        name: logs-generic.otel-default
+  - set: { data_streams.0.indices.0.index_name: datastream-backing-index }
+  - do:
+      indices.get_mapping:
+        index: $datastream-backing-index
+  - is_true: $datastream-backing-index
+  - match: { .$datastream-backing-index.mappings.properties.body.properties.structured.properties.foo\.bar.type: "keyword" }
+---
+Structured log body:
+  - do:
+      bulk:
+        index: logs-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - "@timestamp": 2024-07-18T14:48:33.467654000Z
+            resource:
+              attributes:
+                service.name: my-service
+            body:
+              flattened:
+                foo:
+                  bar: baz
+  - is_false: errors
+  - do:
+      indices.get_data_stream:
+        name: logs-generic.otel-default
+  - set: { data_streams.0.indices.0.index_name: datastream-backing-index }
+  - do:
+      indices.get_mapping:
+        index: $datastream-backing-index
+  - is_true: $datastream-backing-index
+  - match: { .$datastream-backing-index.mappings.properties.body.properties.flattened.type: "flattened" }