From 5f53e253c61a7ea4b603e6bdc8ee8abf31ec8560 Mon Sep 17 00:00:00 2001 From: Philipp Kahr Date: Wed, 17 Sep 2025 12:59:15 +0200 Subject: [PATCH 1/3] Update mapped pages --- manage-data/ingest/transform-enrich/error-handling.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/manage-data/ingest/transform-enrich/error-handling.md b/manage-data/ingest/transform-enrich/error-handling.md index adda77be93..31a6d660a7 100644 --- a/manage-data/ingest/transform-enrich/error-handling.md +++ b/manage-data/ingest/transform-enrich/error-handling.md @@ -1,6 +1,4 @@ --- -mapped_pages: - - https://www.elastic.co/docs/manage-data/ingest/transform-enrich/error-handling.html applies_to: stack: ga serverless: ga From e8171ff5d58c15e7ffffd9888190ecf8ce0b5c45 Mon Sep 17 00:00:00 2001 From: Philipp Kahr Date: Wed, 17 Sep 2025 13:01:51 +0200 Subject: [PATCH 2/3] Fixed mapping and slightly re-arranged the ingest lag headings --- .../ingest/transform-enrich/ingest-lag.md | 25 ++++++++----------- .../readable-maintainable-ingest-pipelines.md | 2 -- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/manage-data/ingest/transform-enrich/ingest-lag.md b/manage-data/ingest/transform-enrich/ingest-lag.md index 46a950d394..be9bd5a58a 100644 --- a/manage-data/ingest/transform-enrich/ingest-lag.md +++ b/manage-data/ingest/transform-enrich/ingest-lag.md @@ -1,6 +1,4 @@ --- -mapped_pages: - - https://www.elastic.co/docs/manage-data/ingest/transform-enrich/calculate-ingest-lag.html applies_to: stack: ga serverless: ga @@ -99,16 +97,6 @@ As discussed above `@timestamp` is set to the timestamp from within the collecte Regardless of the chosen architecture, add a `remove` processor at the end of the pipeline to drop the `_tmp` field. The raw timestamps from the various processing steps are not needed, as the latency in seconds should be sufficient. For additional pipeline architectures, refer to [Ingest architectures](../ingest-reference-architectures.md). -## Logstash - -When Logstash is added to the architecture we must add a timestamp, this can only be done by using Ruby and the simplest form is this: - -``` -ruby { - code => "event.set('[_tmp][logstash_seen]', Time.now());" -} -``` - ### Elastic Agent to Elasticsearch Use `@timestamp` and `event.ingested` to calculate the difference. This will give you the following document. The `event.ingestion.latency` is in seconds. @@ -156,8 +144,17 @@ POST _ingest/pipeline/_simulate } } ``` +### Logstash + +When Logstash is added to the architecture we must add a timestamp, this can only be done by using Ruby and the simplest form is this: + +``` +ruby { + code => "event.set('[_tmp][logstash_seen]', Time.now());" +} +``` -### Elastic Agent => Logstash => Elasticsearch +#### Elastic Agent => Logstash => Elasticsearch Elastic Agent populates the `@timestamp` field, but Logstash doesn't add any timestamp by default. Add a temporary timestamp, for example by setting `_tmp.logstash_seen`. With this, you can calculate the following latency values: @@ -236,7 +233,7 @@ POST _ingest/pipeline/_simulate } ``` -### Elastic Agent => Logstash => Kafka => Logstash => Elasticsearch +#### Elastic Agent => Logstash => Kafka => Logstash => Elasticsearch As with the previous scenario, adding an additional hop introduces another point where latency can occur. The recommendation is to add another temporary timestamp field. For more details, refer to the previous section. diff --git a/manage-data/ingest/transform-enrich/readable-maintainable-ingest-pipelines.md b/manage-data/ingest/transform-enrich/readable-maintainable-ingest-pipelines.md index 57cfe087bb..23c9204a17 100644 --- a/manage-data/ingest/transform-enrich/readable-maintainable-ingest-pipelines.md +++ b/manage-data/ingest/transform-enrich/readable-maintainable-ingest-pipelines.md @@ -1,6 +1,4 @@ --- -mapped_pages: - - https://www.elastic.co/docs/manage-data/ingest/transform-enrich/common-mistakes.html applies_to: stack: ga serverless: ga From 746ee76e9976fc4f07ecccfa238c21439c9e13cb Mon Sep 17 00:00:00 2001 From: Philipp Kahr Date: Wed, 17 Sep 2025 13:08:30 +0200 Subject: [PATCH 3/3] Rework the headings --- manage-data/ingest/transform-enrich/ingest-lag.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manage-data/ingest/transform-enrich/ingest-lag.md b/manage-data/ingest/transform-enrich/ingest-lag.md index be9bd5a58a..d70fbe22fa 100644 --- a/manage-data/ingest/transform-enrich/ingest-lag.md +++ b/manage-data/ingest/transform-enrich/ingest-lag.md @@ -154,7 +154,7 @@ ruby { } ``` -#### Elastic Agent => Logstash => Elasticsearch +### Elastic Agent => Logstash => Elasticsearch Elastic Agent populates the `@timestamp` field, but Logstash doesn't add any timestamp by default. Add a temporary timestamp, for example by setting `_tmp.logstash_seen`. With this, you can calculate the following latency values: @@ -233,7 +233,7 @@ POST _ingest/pipeline/_simulate } ``` -#### Elastic Agent => Logstash => Kafka => Logstash => Elasticsearch +### Elastic Agent => Logstash => Kafka => Logstash => Elasticsearch As with the previous scenario, adding an additional hop introduces another point where latency can occur. The recommendation is to add another temporary timestamp field. For more details, refer to the previous section.