[otel-data] Add @timestamp as sort field for logs and traces (#114049) (#114125)

* Add @timestamp as sort field for logs and traces

* Change order of sort fields

* Adapt tests

Author: gregkalapos

x-pack/plugin/otel-data/src/main/resources/component-templates/[email protected]

@@ -8,7 +8,7 @@ template:
     index:
       mode: logsdb
       sort:
-        field: [ "resource.attributes.host.name" ]
+        field: [ "resource.attributes.host.name", "@timestamp" ]
   mappings:
     properties:
       attributes:

x-pack/plugin/otel-data/src/main/resources/component-templates/[email protected]

@@ -8,7 +8,7 @@ template:
     index:
       mode: logsdb
       sort:
-        field: [ "resource.attributes.host.name" ]
+        field: [ "resource.attributes.host.name", "@timestamp" ]
   mappings:
     _source:
       mode: synthetic

x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml

@@ -70,3 +70,23 @@ setup:
   - match: { hits.hits.0.fields.error\.exception\.type: ["MyException"] }
   - match: { hits.hits.0.fields.error\.exception\.message: ["foo"] }
   - match: { hits.hits.0.fields.error\.stack_trace: ["Exception in thread \"main\" java.lang.RuntimeException: Test exception\n at com.example.GenerateTrace.methodB(GenerateTrace.java:13)\n at com.example.GenerateTrace.methodA(GenerateTrace.java:9)\n at com.example.GenerateTrace.main(GenerateTrace.java:5)"] }
+---
+"resource.attributes.host.name @timestamp should be used as sort fields":
+  - do:
+      bulk:
+        index: logs-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - '{"@timestamp":"2024-07-18T14:49:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "body_text":"error1"}'
+  - is_false: errors
+  - do:
+      indices.get_data_stream:
+        name: logs-generic.otel-default
+  - set: { data_streams.0.indices.0.index_name: datastream-backing-index }
+  - do:
+      indices.get_settings:
+        index: $datastream-backing-index
+  - is_true: $datastream-backing-index
+  - match: { .$datastream-backing-index.settings.index.sort.field.0: "resource.attributes.host.name" }
+  - match: { .$datastream-backing-index.settings.index.sort.field.1: "@timestamp" }

x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml

@@ -76,23 +76,6 @@ setup:
   - match: { hits.hits.0._source.links.1.trace_id: "4aaa9f33312b3dbb8b2c2c62bb7abe1a1"  }
   - match: { hits.hits.0._source.links.1.span_id: "b3b7d1f1f1b4e1e1"  }
 ---
-"Default data_stream.type must be traces":
-  - do:
-      bulk:
-        index: traces-generic.otel-default
-        refresh: true
-        body:
-          - create: {}
-          - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","type":"traces","namespace":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}'
-  - is_false: errors
-  - do:
-      search:
-        index: traces-generic.otel-default
-        body:
-          fields: ["data_stream.type"]
-  - length: { hits.hits: 1 }
-  - match: { hits.hits.0.fields.data_stream\.type: ["traces"] }
----
 Conflicting attribute types:
   - do:
       bulk:
@@ -117,3 +100,23 @@ Conflicting attribute types:
   - length: { hits.hits: 2 }
   - match: { hits.hits.0.fields.attributes\.http\.status_code: [200] }
   - match: { hits.hits.1._ignored: ["attributes.http.status_code"] }
+---
+"resource.attributes.host.name @timestamp should be used as sort fields":
+  - do:
+      bulk:
+        index: traces-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - '{"@timestamp":"2024-07-18T14:49:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "span_id":"1"}'
+  - is_false: errors
+  - do:
+      indices.get_data_stream:
+        name: traces-generic.otel-default
+  - set: { data_streams.0.indices.0.index_name: datastream-backing-index }
+  - do:
+      indices.get_settings:
+        index: $datastream-backing-index
+  - is_true: $datastream-backing-index
+  - match: { .$datastream-backing-index.settings.index.sort.field.0: "resource.attributes.host.name" }
+  - match: { .$datastream-backing-index.settings.index.sort.field.1: "@timestamp" }