[Cloudflare Logpush] Set Datetime value to @timestamp field (#13400)

cloudflare_logpush: Improved the pipeline to set the value of Datetime field to @timestamp.

For cloudflare_logpush.http_request datastream, added a set processor 
to set @timestamp value if EdgeStartTimestamp is not present, 
but Datetime is present.

Author: moxarth-rathod

packages/cloudflare_logpush/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.37.1"
+  changes:
+    - description: Fix handling of http_request events missing `EdgeStartTimestamp`.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/13400
 - version: "1.37.0"
   changes:
     - description: Add parse for missing field on Firewall Event dataset.

packages/cloudflare_logpush/data_stream/http_request/_dev/test/pipeline/test-pipeline-http-request.log-expected.json

@@ -1286,6 +1286,7 @@
             }
         },
         {
+            "@timestamp": "2025-03-24T19:10:40.000Z",
             "cloudflare_logpush": {
                 "http_request": {
                     "client": {
@@ -1296,6 +1297,7 @@
                             "path": "/ping"
                         }
                     },
+                    "datetime": "2025-03-24T19:10:40.000Z",
                     "edge": {
                         "response": {
                             "status": 200

packages/cloudflare_logpush/data_stream/http_request/elasticsearch/ingest_pipeline/default.yml

@@ -56,6 +56,23 @@ processors:
       - append:
           field: error.message
           value: "{{{_ingest.on_failure_message}}}"
+  - date:
+      field: json.Datetime
+      if: ctx.json?.Datetime != null && ctx.json.Datetime != ''
+      formats:
+        - UNIX_MS
+        - ISO8601
+        - yyyy-MM-dd'T'HH:mm:ssZ
+      timezone: UTC
+      target_field: cloudflare_logpush.http_request.datetime
+      on_failure:
+      - append:
+          field: error.message
+          value: "{{{_ingest.on_failure_message}}}"
+  - set:
+      field: '@timestamp'
+      copy_from: cloudflare_logpush.http_request.datetime
+      ignore_empty_value: true
   - set:
       field: '@timestamp'
       copy_from: cloudflare_logpush.http_request.edge.start_time

packages/cloudflare_logpush/data_stream/http_request/fields/fields.yml

@@ -448,6 +448,9 @@
         - name: name
           type: keyword
           description: The human-readable name of the zone.
+    - name: datetime
+      type: date
+      description: Timestamp when the request was received
 - name: log.source.address
   type: keyword
   description: Source address from which the log event was read / sent from.

packages/cloudflare_logpush/docs/README.md

@@ -2777,6 +2777,7 @@ An example event for `http_request` looks as following:
 | cloudflare_logpush.http_request.content_scan.sizes | List of content object sizes. | long |
 | cloudflare_logpush.http_request.content_scan.types | List of content types. | keyword |
 | cloudflare_logpush.http_request.cookies | String key-value pairs for Cookies. | flattened |
+| cloudflare_logpush.http_request.datetime | Timestamp when the request was received | date |
 | cloudflare_logpush.http_request.edge.cf_connecting_o2o | True if the request looped through multiple zones on the Cloudflare edge. | boolean |
 | cloudflare_logpush.http_request.edge.colo.code | IATA airport code of data center that received the request. | keyword |
 | cloudflare_logpush.http_request.edge.colo.id | Cloudflare edge colo id. | long |

packages/cloudflare_logpush/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: cloudflare_logpush
 title: Cloudflare Logpush
-version: "1.37.0"
+version: "1.37.1"
 description: Collect and parse logs from Cloudflare API with Elastic Agent.
 type: integration
 categories: