From 71b8459b7a918de009a5b9c4c50c82076c93eb9d Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Fri, 6 Jun 2025 17:18:28 +0100 Subject: [PATCH 01/11] [apm-data] Set event.dataset if empty for logs --- .../ingest-pipelines/logs-apm.app@default-pipeline.yaml | 6 ++++++ .../ingest-pipelines/logs-apm.error@default-pipeline.yaml | 6 ++++++ x-pack/plugin/apm-data/src/main/resources/resources.yaml | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm.app@default-pipeline.yaml b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm.app@default-pipeline.yaml index a1f9565676fd4..7f5644327173f 100644 --- a/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm.app@default-pipeline.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm.app@default-pipeline.yaml @@ -10,6 +10,12 @@ description: | custom pipelines. All built-in processing occurs in the final pipelines. processors: +# Set event.dataset if unset to meet Anomaly Detection requirements +- set: + if: "ctx.data_stream?.dataset != null" + field: event.dataset + value: "{{{data_stream.dataset}}}" + override: false # Invoke user-defined custom pipelines, in ascending order of specificity: - pipeline: name: global@custom diff --git a/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm.error@default-pipeline.yaml b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm.error@default-pipeline.yaml index c46a1c1b44f96..95b2605840181 100644 --- a/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm.error@default-pipeline.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm.error@default-pipeline.yaml @@ -10,6 +10,12 @@ description: | custom pipelines. All built-in processing occurs in the final pipelines. processors: +# Set event.dataset if unset to meet Anomaly Detection requirements +- set: + if: "ctx.data_stream?.dataset != null" + field: event.dataset + value: "{{{data_stream.dataset}}}" + override: false # Invoke user-defined custom pipelines, in ascending order of specificity: - pipeline: name: global@custom diff --git a/x-pack/plugin/apm-data/src/main/resources/resources.yaml b/x-pack/plugin/apm-data/src/main/resources/resources.yaml index beca2e5890bc0..ea54fd4c072e9 100644 --- a/x-pack/plugin/apm-data/src/main/resources/resources.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/resources.yaml @@ -1,7 +1,7 @@ # "version" holds the version of the templates and ingest pipelines installed # by xpack-plugin apm-data. This must be increased whenever an existing template or # pipeline is changed, in order for it to be updated on Elasticsearch upgrade. -version: 13 +version: 14 component-templates: # Data lifecycle. From 0923bcd461bdc00316ecc4d486301527d67daf8e Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Mon, 9 Jun 2025 17:39:42 +0100 Subject: [PATCH 02/11] Update docs/changelog/129074.yaml --- docs/changelog/129074.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/129074.yaml diff --git a/docs/changelog/129074.yaml b/docs/changelog/129074.yaml new file mode 100644 index 0000000000000..94e60079ea510 --- /dev/null +++ b/docs/changelog/129074.yaml @@ -0,0 +1,5 @@ +pr: 129074 +summary: "[apm-data] Set `event.dataset` if empty for logs" +area: Data streams +type: bug +issues: [] From 7add32d0051b63836179dd59ad0ae84ccbe9fd85 Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Wed, 11 Jun 2025 14:08:34 +0100 Subject: [PATCH 03/11] Introduce logs-apm@pipeline --- .../index-templates/logs-apm.app@template.yaml | 2 +- .../index-templates/logs-apm.error@template.yaml | 2 +- .../logs-apm.app@default-pipeline.yaml | 6 ------ .../logs-apm.error@default-pipeline.yaml | 6 ------ .../ingest-pipelines/logs-apm@pipeline.yaml | 14 ++++++++++++++ .../apm-data/src/main/resources/resources.yaml | 3 +++ 6 files changed, 19 insertions(+), 14 deletions(-) create mode 100644 x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm@pipeline.yaml diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.app@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.app@template.yaml index ff9fe7df03ab0..9f8bf6d94a37b 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.app@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.app@template.yaml @@ -24,4 +24,4 @@ template: settings: index: default_pipeline: logs-apm.app@default-pipeline - final_pipeline: apm@pipeline + final_pipeline: logs-apm@pipeline diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.error@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.error@template.yaml index 5f510df4ee065..77eb7ac669fdd 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.error@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.error@template.yaml @@ -31,4 +31,4 @@ template: settings: index: default_pipeline: logs-apm.error@default-pipeline - final_pipeline: apm@pipeline + final_pipeline: logs-apm@pipeline diff --git a/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm.app@default-pipeline.yaml b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm.app@default-pipeline.yaml index 7f5644327173f..a1f9565676fd4 100644 --- a/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm.app@default-pipeline.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm.app@default-pipeline.yaml @@ -10,12 +10,6 @@ description: | custom pipelines. All built-in processing occurs in the final pipelines. processors: -# Set event.dataset if unset to meet Anomaly Detection requirements -- set: - if: "ctx.data_stream?.dataset != null" - field: event.dataset - value: "{{{data_stream.dataset}}}" - override: false # Invoke user-defined custom pipelines, in ascending order of specificity: - pipeline: name: global@custom diff --git a/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm.error@default-pipeline.yaml b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm.error@default-pipeline.yaml index 95b2605840181..c46a1c1b44f96 100644 --- a/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm.error@default-pipeline.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm.error@default-pipeline.yaml @@ -10,12 +10,6 @@ description: | custom pipelines. All built-in processing occurs in the final pipelines. processors: -# Set event.dataset if unset to meet Anomaly Detection requirements -- set: - if: "ctx.data_stream?.dataset != null" - field: event.dataset - value: "{{{data_stream.dataset}}}" - override: false # Invoke user-defined custom pipelines, in ascending order of specificity: - pipeline: name: global@custom diff --git a/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm@pipeline.yaml b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm@pipeline.yaml new file mode 100644 index 0000000000000..7cb19b1154682 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm@pipeline.yaml @@ -0,0 +1,14 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + managed: true +description: Built-in ingest pipeline for logs-apm.*-* data streams +processors: +# Set event.dataset if unset to meet Anomaly Detection requirements +- set: + if: "ctx.data_stream?.dataset != null" + field: event.dataset + value: "{{{data_stream.dataset}}}" + override: false +- pipeline: + name: apm@pipeline diff --git a/x-pack/plugin/apm-data/src/main/resources/resources.yaml b/x-pack/plugin/apm-data/src/main/resources/resources.yaml index ea54fd4c072e9..4c754f79a5e73 100644 --- a/x-pack/plugin/apm-data/src/main/resources/resources.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/resources.yaml @@ -97,6 +97,9 @@ ingest-pipelines: - metrics-apm@pipeline: dependencies: - apm@pipeline + - logs-apm@pipeline: + dependencies: + - apm@pipeline lifecycle-policies: - logs-apm.app_logs-default_policy From 48b569b2f775085773431c380b261523a88b151a Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Wed, 11 Jun 2025 16:03:01 +0100 Subject: [PATCH 04/11] Add test --- .../rest-api-spec/test/20_logs_pipeline.yml | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_pipeline.yml diff --git a/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_pipeline.yml b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_pipeline.yml new file mode 100644 index 0000000000000..6c8c56858d556 --- /dev/null +++ b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_pipeline.yml @@ -0,0 +1,52 @@ +--- +setup: + - do: + cluster.health: + wait_for_events: languid +--- +"Test logs-apm.error-* event.dataset field": + - do: + bulk: + index: logs-apm.error-eventdataset + refresh: true + body: + - create: {} + - '{"@timestamp": "2017-06-22", "data_stream": {"type": "log", "dataset: "apm.error", "namespace": "eventdataset"}, "log": {"level": "error"}, "error": {"log": {"message": "loglevel"}, "exception": [{"message": "exception_used"}]}}' + + - create: {} + - '{"@timestamp": "2017-06-22", "data_stream": {"type": "log", "dataset: "apm.error", "namespace": "eventdataset"}, "event": {"dataset": "foo"}, "log": {"level": "error"}, "error": {"log": {"message": "loglevel"}, "exception": [{"message": "exception_used"}]}}' + + - is_false: errors + + - do: + search: + index: logs-apm.error-eventdataset + body: + fields: ["event.dataset"] + - length: { hits.hits: 2 } + - match: { hits.hits.0.fields: { "event.dataset": ["error"] } } + - match: { hits.hits.1.fields: { "event.dataset": ["foo"] } } +--- +"Test logs-apm.app.*-* event.dataset field": + - do: + bulk: + index: logs-apm.app.foo-eventdataset + refresh: true + body: + - create: {} + - '{"@timestamp": "2017-06-22", "data_stream": {"type": "log", "dataset: "apm.app.foo", "namespace": "eventdataset"}, "message": "foo"}' + + - create: {} + - '{"@timestamp": "2017-06-22", "data_stream": {"type": "log", "dataset: "apm.app.foo", "namespace": "eventdataset"}, "event": {"dataset": "foo"}, "message": "foo"}' + + - is_false: errors + + - do: + search: + index: logs-apm.app.foo-eventdataset + body: + fields: ["event.dataset"] + - length: { hits.hits: 2 } + - match: { hits.hits.0.fields: { "event.dataset": ["apm.app.foo"] } } + - match: { hits.hits.1.fields: { "event.dataset": ["foo"] } } + From 4bab03d80d767475ee829490ea1c025be3db8058 Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Wed, 11 Jun 2025 16:11:31 +0100 Subject: [PATCH 05/11] Fix test --- .../main/java/org/elasticsearch/test/rest/ESRestTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index e6022779d5297..e46802d7aed7a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -2362,7 +2362,7 @@ protected static boolean isXPackIngestPipeline(String id) { } return switch (id) { case "logs-default-pipeline", "logs@default-pipeline", "logs@json-message", "logs@json-pipeline" -> true; - case "apm@pipeline", "traces-apm@pipeline", "metrics-apm@pipeline" -> true; + case "apm@pipeline", "traces-apm@pipeline", "metrics-apm@pipeline", "logs-apm@pipeline" -> true; case "behavioral_analytics-events-final_pipeline", "ent-search-generic-ingestion", "search-default-ingestion" -> true; case "reindex-data-stream-pipeline" -> true; default -> false; From b584c7f28c1ebee76174c08633986238632f8fab Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Wed, 11 Jun 2025 16:17:17 +0100 Subject: [PATCH 06/11] Bump version after rebase --- x-pack/plugin/apm-data/src/main/resources/resources.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/apm-data/src/main/resources/resources.yaml b/x-pack/plugin/apm-data/src/main/resources/resources.yaml index 4c754f79a5e73..020275783ba3f 100644 --- a/x-pack/plugin/apm-data/src/main/resources/resources.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/resources.yaml @@ -1,7 +1,7 @@ # "version" holds the version of the templates and ingest pipelines installed # by xpack-plugin apm-data. This must be increased whenever an existing template or # pipeline is changed, in order for it to be updated on Elasticsearch upgrade. -version: 14 +version: 15 component-templates: # Data lifecycle. From 57fe0ea38ff83061ae84498d0b7a8a87bf013d3d Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Wed, 11 Jun 2025 16:38:07 +0100 Subject: [PATCH 07/11] Fix typo in test --- .../resources/rest-api-spec/test/20_logs_pipeline.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_pipeline.yml b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_pipeline.yml index 6c8c56858d556..d3bb0c6393df9 100644 --- a/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_pipeline.yml +++ b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_pipeline.yml @@ -11,10 +11,10 @@ setup: refresh: true body: - create: {} - - '{"@timestamp": "2017-06-22", "data_stream": {"type": "log", "dataset: "apm.error", "namespace": "eventdataset"}, "log": {"level": "error"}, "error": {"log": {"message": "loglevel"}, "exception": [{"message": "exception_used"}]}}' + - '{"@timestamp": "2017-06-22", "data_stream": {"type": "log", "dataset": "apm.error", "namespace": "eventdataset"}, "log": {"level": "error"}, "error": {"log": {"message": "loglevel"}, "exception": [{"message": "exception_used"}]}}' - create: {} - - '{"@timestamp": "2017-06-22", "data_stream": {"type": "log", "dataset: "apm.error", "namespace": "eventdataset"}, "event": {"dataset": "foo"}, "log": {"level": "error"}, "error": {"log": {"message": "loglevel"}, "exception": [{"message": "exception_used"}]}}' + - '{"@timestamp": "2017-06-22", "data_stream": {"type": "log", "dataset": "apm.error", "namespace": "eventdataset"}, "event": {"dataset": "foo"}, "log": {"level": "error"}, "error": {"log": {"message": "loglevel"}, "exception": [{"message": "exception_used"}]}}' - is_false: errors @@ -34,10 +34,10 @@ setup: refresh: true body: - create: {} - - '{"@timestamp": "2017-06-22", "data_stream": {"type": "log", "dataset: "apm.app.foo", "namespace": "eventdataset"}, "message": "foo"}' + - '{"@timestamp": "2017-06-22", "data_stream": {"type": "log", "dataset": "apm.app.foo", "namespace": "eventdataset"}, "message": "foo"}' - create: {} - - '{"@timestamp": "2017-06-22", "data_stream": {"type": "log", "dataset: "apm.app.foo", "namespace": "eventdataset"}, "event": {"dataset": "foo"}, "message": "foo"}' + - '{"@timestamp": "2017-06-22", "data_stream": {"type": "log", "dataset": "apm.app.foo", "namespace": "eventdataset"}, "event": {"dataset": "foo"}, "message": "foo"}' - is_false: errors From a0ec35a9618874e12402a8cdf0b4580e21b9c65d Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Wed, 11 Jun 2025 17:17:23 +0100 Subject: [PATCH 08/11] Fix tests again --- .../resources/rest-api-spec/test/20_logs_pipeline.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_pipeline.yml b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_pipeline.yml index d3bb0c6393df9..c9188cffd32b5 100644 --- a/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_pipeline.yml +++ b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_pipeline.yml @@ -11,10 +11,10 @@ setup: refresh: true body: - create: {} - - '{"@timestamp": "2017-06-22", "data_stream": {"type": "log", "dataset": "apm.error", "namespace": "eventdataset"}, "log": {"level": "error"}, "error": {"log": {"message": "loglevel"}, "exception": [{"message": "exception_used"}]}}' + - '{"@timestamp": "2017-06-22", "data_stream": {"type": "logs", "dataset": "apm.error", "namespace": "eventdataset"}, "log": {"level": "error"}, "error": {"log": {"message": "loglevel"}, "exception": [{"message": "exception_used"}]}}' - create: {} - - '{"@timestamp": "2017-06-22", "data_stream": {"type": "log", "dataset": "apm.error", "namespace": "eventdataset"}, "event": {"dataset": "foo"}, "log": {"level": "error"}, "error": {"log": {"message": "loglevel"}, "exception": [{"message": "exception_used"}]}}' + - '{"@timestamp": "2017-06-22", "data_stream": {"type": "logs", "dataset": "apm.error", "namespace": "eventdataset"}, "event": {"dataset": "foo"}, "log": {"level": "error"}, "error": {"log": {"message": "loglevel"}, "exception": [{"message": "exception_used"}]}}' - is_false: errors @@ -24,7 +24,7 @@ setup: body: fields: ["event.dataset"] - length: { hits.hits: 2 } - - match: { hits.hits.0.fields: { "event.dataset": ["error"] } } + - match: { hits.hits.0.fields: { "event.dataset": ["apm.error"] } } - match: { hits.hits.1.fields: { "event.dataset": ["foo"] } } --- "Test logs-apm.app.*-* event.dataset field": @@ -34,10 +34,10 @@ setup: refresh: true body: - create: {} - - '{"@timestamp": "2017-06-22", "data_stream": {"type": "log", "dataset": "apm.app.foo", "namespace": "eventdataset"}, "message": "foo"}' + - '{"@timestamp": "2017-06-22", "data_stream": {"type": "logs", "dataset": "apm.app.foo", "namespace": "eventdataset"}, "message": "foo"}' - create: {} - - '{"@timestamp": "2017-06-22", "data_stream": {"type": "log", "dataset": "apm.app.foo", "namespace": "eventdataset"}, "event": {"dataset": "foo"}, "message": "foo"}' + - '{"@timestamp": "2017-06-22", "data_stream": {"type": "logs", "dataset": "apm.app.foo", "namespace": "eventdataset"}, "event": {"dataset": "foo"}, "message": "foo"}' - is_false: errors From a4c1f16dc98e11fc3c5f965f557e6346e112bd48 Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Thu, 12 Jun 2025 10:04:01 +0100 Subject: [PATCH 09/11] Use copy_from and ignore_empty_value --- .../main/resources/ingest-pipelines/logs-apm@pipeline.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm@pipeline.yaml b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm@pipeline.yaml index 7cb19b1154682..9183eeb8eef4a 100644 --- a/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm@pipeline.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/logs-apm@pipeline.yaml @@ -6,9 +6,9 @@ description: Built-in ingest pipeline for logs-apm.*-* data streams processors: # Set event.dataset if unset to meet Anomaly Detection requirements - set: - if: "ctx.data_stream?.dataset != null" field: event.dataset - value: "{{{data_stream.dataset}}}" + copy_from: "data_stream.dataset" + ignore_empty_value: true override: false - pipeline: name: apm@pipeline From 957d9ef67bfdd923b46dc819c99b91f1c47c087e Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Thu, 12 Jun 2025 10:20:05 +0100 Subject: [PATCH 10/11] Add tests for missing data_stream.dataset --- .../rest-api-spec/test/20_logs_pipeline.yml | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_pipeline.yml b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_pipeline.yml index c9188cffd32b5..268d8415748cd 100644 --- a/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_pipeline.yml +++ b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_pipeline.yml @@ -10,11 +10,18 @@ setup: index: logs-apm.error-eventdataset refresh: true body: + # data_stream.dataset present, event.dataset not present - create: {} - '{"@timestamp": "2017-06-22", "data_stream": {"type": "logs", "dataset": "apm.error", "namespace": "eventdataset"}, "log": {"level": "error"}, "error": {"log": {"message": "loglevel"}, "exception": [{"message": "exception_used"}]}}' - + # data_stream.dataset present, event.dataset present - create: {} - '{"@timestamp": "2017-06-22", "data_stream": {"type": "logs", "dataset": "apm.error", "namespace": "eventdataset"}, "event": {"dataset": "foo"}, "log": {"level": "error"}, "error": {"log": {"message": "loglevel"}, "exception": [{"message": "exception_used"}]}}' + # unlikely: data_stream.dataset not present, event.dataset not present + - create: {} + - '{"@timestamp": "2017-06-22", "log": {"level": "error"}, "error": {"log": {"message": "loglevel"}, "exception": [{"message": "exception_used"}]}}' + # unlikely: data_stream.dataset not present, event.dataset present + - create: {} + - '{"@timestamp": "2017-06-22", "event": {"dataset": "foo"}, "log": {"level": "error"}, "error": {"log": {"message": "loglevel"}, "exception": [{"message": "exception_used"}]}}' - is_false: errors @@ -23,9 +30,11 @@ setup: index: logs-apm.error-eventdataset body: fields: ["event.dataset"] - - length: { hits.hits: 2 } + - length: { hits.hits: 4 } - match: { hits.hits.0.fields: { "event.dataset": ["apm.error"] } } - match: { hits.hits.1.fields: { "event.dataset": ["foo"] } } + - match: { hits.hits.2.fields: null } + - match: { hits.hits.3.fields: { "event.dataset": ["foo"] } } --- "Test logs-apm.app.*-* event.dataset field": - do: @@ -33,11 +42,18 @@ setup: index: logs-apm.app.foo-eventdataset refresh: true body: + # data_stream.dataset present, event.dataset not present - create: {} - '{"@timestamp": "2017-06-22", "data_stream": {"type": "logs", "dataset": "apm.app.foo", "namespace": "eventdataset"}, "message": "foo"}' - + # data_stream.dataset present, event.dataset present - create: {} - '{"@timestamp": "2017-06-22", "data_stream": {"type": "logs", "dataset": "apm.app.foo", "namespace": "eventdataset"}, "event": {"dataset": "foo"}, "message": "foo"}' + # unlikely: data_stream.dataset not present, event.dataset not present + - create: {} + - '{"@timestamp": "2017-06-22", "message": "foo"}' + # unlikely: data_stream.dataset not present, event.dataset present + - create: {} + - '{"@timestamp": "2017-06-22", "event": {"dataset": "foo"}, "message": "foo"}' - is_false: errors @@ -46,7 +62,9 @@ setup: index: logs-apm.app.foo-eventdataset body: fields: ["event.dataset"] - - length: { hits.hits: 2 } + - length: { hits.hits: 4 } - match: { hits.hits.0.fields: { "event.dataset": ["apm.app.foo"] } } - match: { hits.hits.1.fields: { "event.dataset": ["foo"] } } + - match: { hits.hits.2.fields: null } + - match: { hits.hits.3.fields: { "event.dataset": ["foo"] } } From 74f69495e7bff3f71acbfd2fe78ee20246ba44b6 Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Mon, 16 Jun 2025 13:39:48 +0100 Subject: [PATCH 11/11] Bump version --- x-pack/plugin/apm-data/src/main/resources/resources.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/apm-data/src/main/resources/resources.yaml b/x-pack/plugin/apm-data/src/main/resources/resources.yaml index 020275783ba3f..d76fd19f57d0d 100644 --- a/x-pack/plugin/apm-data/src/main/resources/resources.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/resources.yaml @@ -1,7 +1,7 @@ # "version" holds the version of the templates and ingest pipelines installed # by xpack-plugin apm-data. This must be increased whenever an existing template or # pipeline is changed, in order for it to be updated on Elasticsearch upgrade. -version: 15 +version: 16 component-templates: # Data lifecycle.