diff --git a/.chloggen/elasticsearchexporter_metrics_timestamp_number.yaml b/.chloggen/elasticsearchexporter_metrics_timestamp_number.yaml new file mode 100644 index 0000000000000..339631d7a80d0 --- /dev/null +++ b/.chloggen/elasticsearchexporter_metrics_timestamp_number.yaml @@ -0,0 +1,28 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: elasticsearchexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Serialize the `@timestamp` field for metrics as a number (epoch millis) instead of a string (epoch mills with fractional). + This improves the ingestion performance in Elasticsearch as it can leverage an optimized code path for date parsing. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [41811] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/elasticsearchexporter/exporter_test.go b/exporter/elasticsearchexporter/exporter_test.go index b006b18dafd17..634543fcf3d50 100644 --- a/exporter/elasticsearchexporter/exporter_test.go +++ b/exporter/elasticsearchexporter/exporter_test.go @@ -1313,19 +1313,19 @@ func TestExporterMetrics(t *testing.T) { expected := []itemRequest{ { Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.metric.foo":"histogram"}}}`), - Document: []byte(`{"@timestamp":"0.0","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"metrics":{"metric.foo":{"counts":[1,2,3,4],"values":[0.5,1.5,2.5,3.0]}},"resource":{},"scope":{},"_metric_names_hash":"b23939f78dc5f649"}`), + Document: []byte(`{"@timestamp":0,"data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"metrics":{"metric.foo":{"counts":[1,2,3,4],"values":[0.5,1.5,2.5,3.0]}},"resource":{},"scope":{},"_metric_names_hash":"b23939f78dc5f649"}`), }, { Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.metric.foo":"histogram"}}}`), - Document: []byte(`{"@timestamp":"3600000.0","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"metrics":{"metric.foo":{"counts":[4,5,6,7],"values":[2.0,4.5,5.5,6.0]}},"resource":{},"scope":{},"_metric_names_hash":"b23939f78dc5f649"}`), + Document: []byte(`{"@timestamp":3600000,"data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"metrics":{"metric.foo":{"counts":[4,5,6,7],"values":[2.0,4.5,5.5,6.0]}},"resource":{},"scope":{},"_metric_names_hash":"b23939f78dc5f649"}`), }, { Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.metric.sum":"gauge_double"}}}`), - Document: []byte(`{"@timestamp":"3600000.0","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"metrics":{"metric.sum":1.5},"resource":{},"scope":{},"start_timestamp":"7200000.0","_metric_names_hash":"f4a8ac5e1b330ad6"}`), + Document: []byte(`{"@timestamp":3600000,"data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"metrics":{"metric.sum":1.5},"resource":{},"scope":{},"start_timestamp":7200000,"_metric_names_hash":"f4a8ac5e1b330ad6"}`), }, { Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.metric.summary":"summary"}}}`), - Document: []byte(`{"@timestamp":"10800000.0","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"metrics":{"metric.summary":{"sum":1.5,"value_count":1}},"resource":{},"scope":{},"start_timestamp":"10800000.0","_metric_names_hash":"2f30c89222c9d308"}`), + Document: []byte(`{"@timestamp":10800000,"data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"metrics":{"metric.summary":{"sum":1.5,"value_count":1}},"resource":{},"scope":{},"start_timestamp":10800000,"_metric_names_hash":"2f30c89222c9d308"}`), }, } @@ -1435,7 +1435,7 @@ func TestExporterMetrics(t *testing.T) { expected := []itemRequest{ { Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.sum":"gauge_long","metrics.summary":"summary"}}}`), - Document: []byte(`{"@timestamp":"0.0","_doc_count":10,"data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"metrics":{"sum":0,"summary":{"sum":1.0,"value_count":10}},"resource":{},"scope":{},"_metric_names_hash":"e446964dc8337bbb"}`), + Document: []byte(`{"@timestamp":0,"_doc_count":10,"data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"metrics":{"sum":0,"summary":{"sum":1.0,"value_count":10}},"resource":{},"scope":{},"_metric_names_hash":"e446964dc8337bbb"}`), }, } @@ -1485,11 +1485,11 @@ func TestExporterMetrics(t *testing.T) { expected := []itemRequest{ { Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.histogram.summary":"summary"}}}`), - Document: []byte(`{"@timestamp":"0.0","_doc_count":10,"data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{},"metrics":{"histogram.summary":{"sum":1.0,"value_count":10}},"resource":{},"scope":{},"_metric_names_hash":"fcd1d6737d725996"}`), + Document: []byte(`{"@timestamp":0,"_doc_count":10,"data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{},"metrics":{"histogram.summary":{"sum":1.0,"value_count":10}},"resource":{},"scope":{},"_metric_names_hash":"fcd1d6737d725996"}`), }, { Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.exphistogram.summary":"summary"}}}`), - Document: []byte(`{"@timestamp":"3600000.0","_doc_count":10,"data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{},"metrics":{"exphistogram.summary":{"sum":1.0,"value_count":10}},"resource":{},"scope":{},"_metric_names_hash":"6a10ca190ae63c5"}`), + Document: []byte(`{"@timestamp":3600000,"_doc_count":10,"data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{},"metrics":{"exphistogram.summary":{"sum":1.0,"value_count":10}},"resource":{},"scope":{},"_metric_names_hash":"6a10ca190ae63c5"}`), }, } @@ -1528,7 +1528,7 @@ func TestExporterMetrics(t *testing.T) { expected := []itemRequest{ { Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.foo.bar":"gauge_long","metrics.foo":"gauge_long","metrics.foo.bar.baz":"gauge_long"}}}`), - Document: []byte(`{"@timestamp":"0.0","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"metrics":{"foo":0,"foo.bar":0,"foo.bar.baz":0},"resource":{},"scope":{},"_metric_names_hash":"9c732a69b35274fe"}`), + Document: []byte(`{"@timestamp":0,"data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"metrics":{"foo":0,"foo.bar":0,"foo.bar.baz":0},"resource":{},"scope":{},"_metric_names_hash":"9c732a69b35274fe"}`), }, } diff --git a/exporter/elasticsearchexporter/internal/serializer/otelserializer/common.go b/exporter/elasticsearchexporter/internal/serializer/otelserializer/common.go index f2eb629bd0773..f997b77c5b42d 100644 --- a/exporter/elasticsearchexporter/internal/serializer/otelserializer/common.go +++ b/exporter/elasticsearchexporter/internal/serializer/otelserializer/common.go @@ -165,6 +165,11 @@ func writeTimestampField(v *json.Visitor, key string, timestamp pcommon.Timestam _ = v.OnString(strconv.FormatUint(msec, 10) + "." + strconv.FormatUint(nsec, 10)) } +func writeTimestampEpochMillisField(v *json.Visitor, key string, timestamp pcommon.Timestamp) { + _ = v.OnKey(key) + _ = v.OnUint64(uint64(timestamp) / 1e6) +} + func writeUIntField(v *json.Visitor, key string, i uint64) { _ = v.OnKey(key) _ = v.OnUint64(i) diff --git a/exporter/elasticsearchexporter/internal/serializer/otelserializer/metrics.go b/exporter/elasticsearchexporter/internal/serializer/otelserializer/metrics.go index b6834b25cf916..68221712a27ac 100644 --- a/exporter/elasticsearchexporter/internal/serializer/otelserializer/metrics.go +++ b/exporter/elasticsearchexporter/internal/serializer/otelserializer/metrics.go @@ -30,9 +30,9 @@ func (*Serializer) SerializeMetrics(resource pcommon.Resource, resourceSchemaURL // This is required to generate the correct dynamic mapping in ES. v.SetExplicitRadixPoint(true) _ = v.OnObjectStart(-1, structform.AnyType) - writeTimestampField(v, "@timestamp", dp0.Timestamp()) + writeTimestampEpochMillisField(v, "@timestamp", dp0.Timestamp()) if dp0.StartTimestamp() != 0 { - writeTimestampField(v, "start_timestamp", dp0.StartTimestamp()) + writeTimestampEpochMillisField(v, "start_timestamp", dp0.StartTimestamp()) } writeStringFieldSkipDefault(v, "unit", dp0.Metric().Unit()) writeDataStream(v, idx) diff --git a/exporter/elasticsearchexporter/internal/serializer/otelserializer/metrics_test.go b/exporter/elasticsearchexporter/internal/serializer/otelserializer/metrics_test.go index ef212d1cc3feb..7d483ee22aa9c 100644 --- a/exporter/elasticsearchexporter/internal/serializer/otelserializer/metrics_test.go +++ b/exporter/elasticsearchexporter/internal/serializer/otelserializer/metrics_test.go @@ -52,7 +52,7 @@ func TestSerializeMetricsConflict(t *testing.T) { assert.EqualError(t, validationErrors[0], "metric with name 'foo' has already been serialized in document with timestamp 1970-01-01T00:00:00.000000000Z") assert.Equal(t, map[string]any{ - "@timestamp": "0.0", + "@timestamp": json.Number("0"), "resource": map[string]any{}, "scope": map[string]any{}, "metrics": map[string]any{