sentinel_one: Add support for application data stream.

Added support for ingesting data through the SentinelOne application data stream.
This includes necessary configuration changes and input adjustments to enable
collection and parsing of application-related events.

Tested on the live samples collected through the SentinelOne API.

Author: mohitjha-elastic

packages/sentinel_one/_dev/build/docs/README.md

@@ -65,6 +65,14 @@ This is the `alert` dataset.
 
 {{fields "alert"}}
 
+### application
+
+This is the `application` dataset.
+
+{{event "application"}}
+
+{{fields "application"}}
+
 ### group
 
 This is the `group` dataset.

packages/sentinel_one/_dev/deploy/docker/files/config.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.37.0"
+  changes:
+    - description: Add support for application data stream.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/14834
 - version: "1.36.0"
   changes:
     - description: Add configuration option to filter results by Site IDs.

packages/sentinel_one/changelog.yml

@@ -0,0 +1,3 @@
+fields:
+  tags:
+    - preserve_duplicate_custom_fields

packages/sentinel_one/data_stream/application/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,3 @@
+{"accountName":"Elastic","applicationInstallationDate":"2023-04-28T07:15:57Z","applicationInstallationPath":null,"applicationName":"7-Zip","coreCount":4,"cpe":"cpe:2.3:a:7-zip:7-zip:22.1:*:*:*:*:*:*:*","cpuCount":2,"detectionDate":"2025-06-02T04:46:51.710561Z","endpointId":"2162143406517023959","endpointName":"DESKTOP-R1E2DQ2","endpointType":"desktop","endpointUuid":"187d5f3c7af341079041baeb9f88a511","fileSize":5601,"groupName":"Default Group","id":"2228104980080385459","osArch":"64 bit","osName":"Windows 10 Pro","osType":"windows","osVersion":"Windows 10 Pro 19044","siteName":"Default site","version":"22.01"}
+{"accountName":"Elastic","applicationInstallationDate":"2023-04-19T18:30:00Z","applicationInstallationPath":null,"applicationName":"Brave","coreCount":4,"cpe":"cpe:2.3:a:brave:brave:112.1.50.121:*:*:*:*:*:*:*","cpuCount":2,"detectionDate":"2025-06-02T04:46:51.710582Z","endpointId":"2162143406517023959","endpointName":"DESKTOP-R1E2DQ2","endpointType":"desktop","endpointUuid":"187d5f3c7af341079041baeb9f88a511","fileSize":null,"groupName":"Default Group","id":"2228104980315266498","osArch":"64 bit","osName":"Windows 10 Pro","osType":"windows","osVersion":"Windows 10 Pro 19044","siteName":"Default site","version":"112.1.50.121"}
+{"accountName":"Elastic","applicationInstallationDate":"2025-03-13T10:45:01Z","applicationInstallationPath":null,"applicationName":"Elastic Agent","coreCount":2,"cpe":"cpe:2.3:a:elastic:elastic_agent:8.17.3:*:*:*:*:*:*:*","cpuCount":1,"detectionDate":"2025-05-19T18:00:51.166610Z","endpointId":"2169705024028266268","endpointName":"srv-win-defend-03","endpointType":"server","endpointUuid":"eb655be8be894dae97711ebb9a9091ae","fileSize":517364,"groupName":"Default Group","id":"2218357748550497214","osArch":"64 bit","osName":"Windows Server 2022 Datacenter","osType":"windows","osVersion":"Windows Server 2022 Datacenter 20348","siteName":"Default site","version":"8.17.3"}

packages/sentinel_one/data_stream/application/_dev/test/pipeline/test-pipeline-application.log

@@ -0,0 +1,188 @@
+{
+    "expected": [
+        {
+            "ecs": {
+                "version": "8.17.0"
+            },
+            "event": {
+                "category": [
+                    "package"
+                ],
+                "kind": "event",
+                "original": "{\"accountName\":\"Elastic\",\"applicationInstallationDate\":\"2023-04-28T07:15:57Z\",\"applicationInstallationPath\":null,\"applicationName\":\"7-Zip\",\"coreCount\":4,\"cpe\":\"cpe:2.3:a:7-zip:7-zip:22.1:*:*:*:*:*:*:*\",\"cpuCount\":2,\"detectionDate\":\"2025-06-02T04:46:51.710561Z\",\"endpointId\":\"2162143406517023959\",\"endpointName\":\"DESKTOP-R1E2DQ2\",\"endpointType\":\"desktop\",\"endpointUuid\":\"187d5f3c7af341079041baeb9f88a511\",\"fileSize\":5601,\"groupName\":\"Default Group\",\"id\":\"2228104980080385459\",\"osArch\":\"64 bit\",\"osName\":\"Windows 10 Pro\",\"osType\":\"windows\",\"osVersion\":\"Windows 10 Pro 19044\",\"siteName\":\"Default site\",\"version\":\"22.01\"}",
+                "type": [
+                    "info"
+                ]
+            },
+            "host": {
+                "name": "DESKTOP-R1E2DQ2",
+                "os": {
+                    "full": "Windows 10 Pro 19044",
+                    "name": "Windows 10 Pro",
+                    "type": "windows"
+                },
+                "type": "desktop"
+            },
+            "package": {
+                "installed": "2023-04-28T07:15:57.000Z",
+                "name": "7-Zip",
+                "size": 5601,
+                "version": "22.01"
+            },
+            "related": {
+                "hosts": [
+                    "DESKTOP-R1E2DQ2"
+                ]
+            },
+            "sentinel_one": {
+                "application": {
+                    "account_name": "Elastic",
+                    "application_installation_date": "2023-04-28T07:15:57.000Z",
+                    "application_name": "7-Zip",
+                    "core_count": 4,
+                    "cpe": "cpe:2.3:a:7-zip:7-zip:22.1:*:*:*:*:*:*:*",
+                    "cpu_count": 2,
+                    "detection_date": "2025-06-02T04:46:51.710Z",
+                    "endpoint_id": "2162143406517023959",
+                    "endpoint_name": "DESKTOP-R1E2DQ2",
+                    "endpoint_type": "desktop",
+                    "endpoint_uuid": "187d5f3c7af341079041baeb9f88a511",
+                    "file_size": 5601,
+                    "group_name": "Default Group",
+                    "id": "2228104980080385459",
+                    "os_arch": "64 bit",
+                    "os_name": "Windows 10 Pro",
+                    "os_type": "windows",
+                    "os_version": "Windows 10 Pro 19044",
+                    "site_name": "Default site",
+                    "version": "22.01"
+                }
+            },
+            "tags": [
+                "preserve_duplicate_custom_fields"
+            ]
+        },
+        {
+            "ecs": {
+                "version": "8.17.0"
+            },
+            "event": {
+                "category": [
+                    "package"
+                ],
+                "kind": "event",
+                "original": "{\"accountName\":\"Elastic\",\"applicationInstallationDate\":\"2023-04-19T18:30:00Z\",\"applicationInstallationPath\":null,\"applicationName\":\"Brave\",\"coreCount\":4,\"cpe\":\"cpe:2.3:a:brave:brave:112.1.50.121:*:*:*:*:*:*:*\",\"cpuCount\":2,\"detectionDate\":\"2025-06-02T04:46:51.710582Z\",\"endpointId\":\"2162143406517023959\",\"endpointName\":\"DESKTOP-R1E2DQ2\",\"endpointType\":\"desktop\",\"endpointUuid\":\"187d5f3c7af341079041baeb9f88a511\",\"fileSize\":null,\"groupName\":\"Default Group\",\"id\":\"2228104980315266498\",\"osArch\":\"64 bit\",\"osName\":\"Windows 10 Pro\",\"osType\":\"windows\",\"osVersion\":\"Windows 10 Pro 19044\",\"siteName\":\"Default site\",\"version\":\"112.1.50.121\"}",
+                "type": [
+                    "info"
+                ]
+            },
+            "host": {
+                "name": "DESKTOP-R1E2DQ2",
+                "os": {
+                    "full": "Windows 10 Pro 19044",
+                    "name": "Windows 10 Pro",
+                    "type": "windows"
+                },
+                "type": "desktop"
+            },
+            "package": {
+                "installed": "2023-04-19T18:30:00.000Z",
+                "name": "Brave",
+                "version": "112.1.50.121"
+            },
+            "related": {
+                "hosts": [
+                    "DESKTOP-R1E2DQ2"
+                ]
+            },
+            "sentinel_one": {
+                "application": {
+                    "account_name": "Elastic",
+                    "application_installation_date": "2023-04-19T18:30:00.000Z",
+                    "application_name": "Brave",
+                    "core_count": 4,
+                    "cpe": "cpe:2.3:a:brave:brave:112.1.50.121:*:*:*:*:*:*:*",
+                    "cpu_count": 2,
+                    "detection_date": "2025-06-02T04:46:51.710Z",
+                    "endpoint_id": "2162143406517023959",
+                    "endpoint_name": "DESKTOP-R1E2DQ2",
+                    "endpoint_type": "desktop",
+                    "endpoint_uuid": "187d5f3c7af341079041baeb9f88a511",
+                    "group_name": "Default Group",
+                    "id": "2228104980315266498",
+                    "os_arch": "64 bit",
+                    "os_name": "Windows 10 Pro",
+                    "os_type": "windows",
+                    "os_version": "Windows 10 Pro 19044",
+                    "site_name": "Default site",
+                    "version": "112.1.50.121"
+                }
+            },
+            "tags": [
+                "preserve_duplicate_custom_fields"
+            ]
+        },
+        {
+            "ecs": {
+                "version": "8.17.0"
+            },
+            "event": {
+                "category": [
+                    "package"
+                ],
+                "kind": "event",
+                "original": "{\"accountName\":\"Elastic\",\"applicationInstallationDate\":\"2025-03-13T10:45:01Z\",\"applicationInstallationPath\":null,\"applicationName\":\"Elastic Agent\",\"coreCount\":2,\"cpe\":\"cpe:2.3:a:elastic:elastic_agent:8.17.3:*:*:*:*:*:*:*\",\"cpuCount\":1,\"detectionDate\":\"2025-05-19T18:00:51.166610Z\",\"endpointId\":\"2169705024028266268\",\"endpointName\":\"srv-win-defend-03\",\"endpointType\":\"server\",\"endpointUuid\":\"eb655be8be894dae97711ebb9a9091ae\",\"fileSize\":517364,\"groupName\":\"Default Group\",\"id\":\"2218357748550497214\",\"osArch\":\"64 bit\",\"osName\":\"Windows Server 2022 Datacenter\",\"osType\":\"windows\",\"osVersion\":\"Windows Server 2022 Datacenter 20348\",\"siteName\":\"Default site\",\"version\":\"8.17.3\"}",
+                "type": [
+                    "info"
+                ]
+            },
+            "host": {
+                "name": "srv-win-defend-03",
+                "os": {
+                    "full": "Windows Server 2022 Datacenter 20348",
+                    "name": "Windows Server 2022 Datacenter",
+                    "type": "windows"
+                },
+                "type": "server"
+            },
+            "package": {
+                "installed": "2025-03-13T10:45:01.000Z",
+                "name": "Elastic Agent",
+                "size": 517364,
+                "version": "8.17.3"
+            },
+            "related": {
+                "hosts": [
+                    "srv-win-defend-03"
+                ]
+            },
+            "sentinel_one": {
+                "application": {
+                    "account_name": "Elastic",
+                    "application_installation_date": "2025-03-13T10:45:01.000Z",
+                    "application_name": "Elastic Agent",
+                    "core_count": 2,
+                    "cpe": "cpe:2.3:a:elastic:elastic_agent:8.17.3:*:*:*:*:*:*:*",
+                    "cpu_count": 1,
+                    "detection_date": "2025-05-19T18:00:51.166Z",
+                    "endpoint_id": "2169705024028266268",
+                    "endpoint_name": "srv-win-defend-03",
+                    "endpoint_type": "server",
+                    "endpoint_uuid": "eb655be8be894dae97711ebb9a9091ae",
+                    "file_size": 517364,
+                    "group_name": "Default Group",
+                    "id": "2218357748550497214",
+                    "os_arch": "64 bit",
+                    "os_name": "Windows Server 2022 Datacenter",
+                    "os_type": "windows",
+                    "os_version": "Windows Server 2022 Datacenter 20348",
+                    "site_name": "Default site",
+                    "version": "8.17.3"
+                }
+            },
+            "tags": [
+                "preserve_duplicate_custom_fields"
+            ]
+        }
+    ]
+}

packages/sentinel_one/data_stream/application/_dev/test/pipeline/test-pipeline-application.log-expected.json

@@ -0,0 +1,12 @@
+input: cel
+service: sentinel_one
+vars:
+  url: http://{{Hostname}}:{{Port}}
+  api_token: xxxx
+data_stream:
+  vars:
+    batch_size: 2
+    preserve_original_event: true
+    preserve_duplicate_custom_fields: true
+assert:
+  hit_count: 4

packages/sentinel_one/data_stream/application/_dev/test/system/test-default-config.yml

@@ -0,0 +1,145 @@
+config_version: 2
+interval: {{interval}}
+resource.tracer:
+  enabled: {{enable_request_tracer}}
+  filename: "../../logs/cel/http-request-trace-*.ndjson"
+  maxbackups: 5
+{{#if proxy_url}}
+resource.proxy_url: {{proxy_url}}
+{{/if}}
+{{#if ssl}}
+resource.ssl: {{ssl}}
+{{/if}}
+{{#if http_client_timeout}}
+resource.timeout: {{http_client_timeout}}
+{{/if}}
+resource.url: {{url}}
+state:
+  api_token: {{api_token}}
+  batch_size: {{batch_size}}
+redact:
+  fields:
+    - api_token
+program: |
+  (
+    has(state.?worklist.data) && size(state.worklist.data) > 0 ?
+      state
+    :
+      state.with(
+        request("GET",
+          state.url.trim_right("/") + "/web/api/v2.1/application-management/inventory?" + {
+            "skipCount": ["true"],
+            "limit": [string(state.batch_size)],
+            ?"cursor": state.?next_page.token.optMap(v, [v]),
+          }.format_query()
+        ).with({
+          "Header":{
+            "Authorization": ["ApiToken " + state.api_token],
+          },
+        }).do_request().as(resp, resp.StatusCode == 200 ?
+          resp.Body.decode_json().as(body,
+            {
+              "worklist": body,
+              "next_page": {
+                ?"token": body.?pagination.nextCursor.orValue(null) != null ? optional.of(body.pagination.nextCursor) : optional.none(),
+              },
+              "fetch_more": body.?pagination.nextCursor.orValue(null) != null,
+            }
+          )
+        :
+          {
+            "events": {
+              "error": {
+                "code": string(resp.StatusCode),
+                "id": string(resp.Status),
+                "message": "GET " + state.url.trim_right("/") + "/web/api/v2.1/application-management/inventory" +
+                (
+                  size(resp.Body) != 0 ?
+                    string(resp.Body)
+                  :
+                    string(resp.Status) + " (" + string(resp.StatusCode) + ")"
+                ),
+              },
+            },
+            "want_more": false,
+            "offset": 0,
+          }
+        )
+      )
+  ).as(state,
+    state.with(
+      !has(state.worklist) ? // Exit early due to GET failure.
+        state
+      : (has(state.worklist.data) && size(state.worklist.data) > 0) ?
+          request("GET",
+            state.url.trim_right("/") + "/web/api/v2.1/application-management/inventory/endpoints?" + {
+              "skipCount": ["true"],
+              "applicationName": [string(state.worklist.data[0].applicationName)],
+              "applicationVendor": [string(state.worklist.data[0].applicationVendor)],
+              "limit": [string(state.batch_size)],
+              ?"cursor": state.?next_chain.token.optMap(v, [v]),
+            }.format_query()
+          ).with({
+            "Header":{
+              "Authorization": ["ApiToken " + state.api_token],
+            }
+          }).do_request().as(resp, resp.StatusCode == 200 ?
+            resp.Body.decode_json().as(body, {
+              "events": (
+                has(body.data) && body.data.size() > 0 ?
+                  body.data.map(e,{
+                    "message": e.encode_json(),
+                  })
+                :
+                  [{"message": "retry"}]
+              ),
+              "next_chain": {
+                ?"token": body.?pagination.nextCursor.orValue(null) != null ? optional.of(body.pagination.nextCursor) : optional.none(),
+              },
+              "worklist": {
+                "data": body.?pagination.nextCursor.orValue(null) != null ? state.worklist.data : tail(state.worklist.data),
+              },
+              "want_more": state.?fetch_more.orValue(false) ? state.fetch_more : body.?pagination.nextCursor.orValue(null) != null,
+            })
+          :
+            {
+              "events": {
+                "error": {
+                  "code": string(resp.StatusCode),
+                  "id": string(resp.Status),
+                  "message": "GET " + state.url.trim_right("/") + "/web/api/v2.1/application-management/inventory/endpoints" +
+                  (
+                    size(resp.Body) != 0 ?
+                      string(resp.Body)
+                    :
+                      string(resp.Status) + " (" + string(resp.StatusCode) + ")"
+                  ),
+                },
+              },
+              "want_more": false,
+            }
+          )
+        :
+          {
+            "events": [],
+            "want_more": false,
+          }
+    )
+  )
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#if preserve_duplicate_custom_fields}}
+  - preserve_duplicate_custom_fields
+{{/if}}
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}