From 7aff4dccebce1d12b248149013f6a1eed0c10619 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Tue, 18 Feb 2025 18:30:37 +0100 Subject: [PATCH] Add _metric_names_hash field to OTel metric mappings (#120952) If metrics that have the same timestamp and dimensions aren't grouped into the same document, ES will consider them to be a duplicate. The _metric_names_hash field will be set by the OTel ES exporter. As it's mapped as a time_series_dimensions, it creates a different _tsid for documents with different sets of metrics. The tradeoff is that if the composition of the metrics grouping changes over time, a different _tsid will be created. That has an impact on the rate aggregation for counters. --- docs/changelog/120952.yaml | 5 ++++ .../metrics-otel@mappings.yaml | 4 ++++ .../rest-api-spec/test/20_metrics_tests.yml | 23 +++++++++++++++++++ 3 files changed, 32 insertions(+) create mode 100644 docs/changelog/120952.yaml diff --git a/docs/changelog/120952.yaml b/docs/changelog/120952.yaml new file mode 100644 index 0000000000000..42d375b979920 --- /dev/null +++ b/docs/changelog/120952.yaml @@ -0,0 +1,5 @@ +pr: 120952 +summary: Add `_metric_names_hash` field to OTel metric mappings +area: Data streams +type: bug +issues: [] diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml index 368d538c304e3..cfc68889e8c9e 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml @@ -14,6 +14,10 @@ template: type: passthrough dynamic: true priority: 10 + # workaround for https://github.com/elastic/elasticsearch/issues/99123 + _metric_names_hash: + type: keyword + time_series_dimension: true unit: type: keyword time_series_dimension: true diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml index 38732b61dbedc..8e4539e445e11 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml @@ -305,3 +305,26 @@ Metrics with different scope names are not duplicates: metrics: foo.bar: 42 - is_false: errors + +--- +Metrics with different metric name hashes are not duplicates: + - do: + bulk: + index: metrics-generic.otel-default + refresh: true + body: + - create: {"dynamic_templates":{"metrics.foo":"counter_long"}} + - "@timestamp": 2024-07-18T14:00:00Z + attributes: + foo: bar + metrics: + foo: 42 + _metric_names_hash: a9f37ed7 + - create: {"dynamic_templates":{"metrics.bar":"counter_long"}} + - "@timestamp": 2024-07-18T14:00:00Z + attributes: + foo: bar + metrics: + bar: 42 + _metric_names_hash: 76b77d1a + - is_false: errors