microsoft_defender_endpoint: add support for oauth endpoint params (elastic#15667)

Add support for the oauth_endpoint_params configuration
parameter for all available data streams.

Log data stream still works under httpjson so the option
has been added under data stream level along with all
the OAuth2 options for this data stream.

For the another data streams, as they work under the
CEL input, it has been added at input level so adding
any value to this option will affect all data streams that
rely on CEL (machine, machine_action, and vulnerability).

Finally, the auth logic for the vulnerability data stream is
implemented in the CEL program instead of delegate in
the CEL auth options for the input.
Therefore, the oauth endpoint params in this case are added
manually in the program as well.

Author: chemamartinez

packages/microsoft_defender_endpoint/_dev/deploy/docker/docker-compose.yml

@@ -7,7 +7,7 @@ services:
       - ${SERVICE_LOGS_DIR}:/var/log
     command: /bin/sh -c "cp /sample_logs/* /var/log/"
   microsoft-defender-mock:
-    image: docker.elastic.co/observability/stream:v0.18.0
+    image: docker.elastic.co/observability/stream:v0.20.0
     ports:
       - 8080
     volumes:
@@ -19,7 +19,7 @@ services:
       - --addr=:8080
       - --config=/config.yml
   microsoft-defender-endpoint-vulnerability-cel:
-    image: docker.elastic.co/observability/stream:v0.18.0
+    image: docker.elastic.co/observability/stream:v0.20.0
     ports:
       - 8080
     volumes:

packages/microsoft_defender_endpoint/_dev/deploy/docker/http-mock-config.yml

@@ -16,6 +16,33 @@ rules:
             - "application/json"
         body: |-
           {"access_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ilg1ZVhrN","token_type": "Bearer","not_before": 1549647431,"expires_in": 3600,"resource": "f2a76e08-93f2-4350-833c-965c02483b11"}
+  # Refresh token to get access token: https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-auth-code-flow#refresh-the-access-token
+  - path: /tenant_id/oauth2/token
+    methods: [ POST ]
+    query_params:
+      refresh_token: refresh_token_123
+      grant_type: refresh_token
+    request_headers:
+      Authorization:
+        - "Basic dGVzdC1hcHAtaWQ6dGVzdC1zZWNyZXQ="
+      Content-Type:
+        - "application/x-www-form-urlencoded"
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/json"
+        body: |-
+          {{ minify_json `
+          {
+            "access_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ilg1ZVhrN",
+            "token_type": "Bearer",
+            "expires_in": 2,
+            "scope": "https%3A%2F%2Fgraph.microsoft.com%2Fmail.read",
+            "refresh_token": "AwABAAAAvPM1KaPlrEqdFSBzjqfTGAMxZGUTdM0t4B4...",
+            "id_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJub25lIn0.eyJhdWQiOiIyZDRkMTFhMi1mODE0LTQ2YTctOD..."
+          }
+          `}}
   - path: /api/alerts
     methods: [GET]
     query_params:
@@ -152,6 +179,34 @@ rules:
             - "application/json"
         body: |-
           {"token_type":"Bearer","expires_in":"3599","ext_expires_in":"3599","expires_on":"1924905600","not_before":"1730182638","resource":"https://management.azure.com/","access_token":"eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ilg1ZVhrN"}
+  # Refresh token to get access token: https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-auth-code-flow#refresh-the-access-token
+  - path: /tenant-id/oauth2/v2.0/token
+    methods: [ POST ]
+    query_params:
+      scope: 'https://securitycenter.onmicrosoft.com/windowsatpservice/.default'
+      refresh_token: refresh_token_123
+      grant_type: refresh_token
+    request_headers:
+      Authorization:
+        - "Basic dGVzdC1jZWwtY2xpZW50LWlkOnRlc3QtY2VsLWNsaWVudC1zZWNyZXQ="
+      Content-Type:
+        - "application/x-www-form-urlencoded"
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/json"
+        body: |-
+          {{ minify_json `
+          {
+            "access_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ilg1ZVhrN",
+            "token_type": "Bearer",
+            "expires_in": 2,
+            "scope": "https%3A%2F%2Fgraph.microsoft.com%2Fmail.read",
+            "refresh_token": "AwABAAAAvPM1KaPlrEqdFSBzjqfTGAMxZGUTdM0t4B4...",
+            "id_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJub25lIn0.eyJhdWQiOiIyZDRkMTFhMi1mODE0LTQ2YTctOD..."
+          }
+          `}}
   - path: /api/machineactions
     methods: ["GET"]
     request_headers:

packages/microsoft_defender_endpoint/_dev/deploy/docker/vulnerability-http-mock-config.yml

@@ -8,6 +8,35 @@ rules:
             - "application/json"
         body: |-
           {"token_type":"Bearer","expires_in":3599,"ext_expires_in":3599,"access_token":"topsecretaccesstokenthatshouldnotbeleakedforabit"}
+  # Refresh token to get access token: https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-auth-code-flow#refresh-the-access-token
+  - path: /tenant_id/oauth2/token
+    methods: [ POST ]
+    query_params:
+      refresh_token:
+        - refresh_token_1
+        - refresh_token_2
+      grant_type: refresh_token
+    request_headers:
+      Authorization:
+        - "Basic dGVzdC1hcHAtaWQ6dGVzdC1zZWNyZXQ="
+      Content-Type:
+        - "application/x-www-form-urlencoded"
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/json"
+        body: |-
+          {{ minify_json `
+          {
+            "access_token": "topsecretaccesstokenthatshouldnotbeleakedforabit",
+            "token_type": "Bearer",
+            "expires_in": 2,
+            "scope": "https%3A%2F%2Fgraph.microsoft.com%2Fmail.read",
+            "refresh_token": "AwABAAAAvPM1KaPlrEqdFSBzjqfTGAMxZGUTdM0t4B4...",
+            "id_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJub25lIn0.eyJhdWQiOiIyZDRkMTFhMi1mODE0LTQ2YTctOD..."
+          }
+          `}}
   - path: /api/machines/SoftwareVulnerabilitiesExport
     methods: ["GET"]
     query_params:

packages/microsoft_defender_endpoint/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "4.1.0"
+  changes:
+    - description: Add support for OAuth2 Endpoint Params option.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15667
 - version: "4.0.0"
   changes:
     - description: |

packages/microsoft_defender_endpoint/data_stream/log/_dev/test/system/test-oauth-params-config.yml

@@ -0,0 +1,15 @@
+service: microsoft-defender-mock
+input: httpjson
+data_stream:
+  vars:
+    client_id: test-app-id
+    client_secret: test-secret
+    tenant_id: tenant_id
+    login_url: http://{{Hostname}}:{{Port}}
+    request_url: http://{{Hostname}}:{{Port}}/api/alerts
+    enable_request_tracer: true
+    oauth_endpoint_params: |
+      grant_type: refresh_token
+      refresh_token: 'refresh_token_123'
+assert:
+  hit_count: 3

packages/microsoft_defender_endpoint/data_stream/log/agent/stream/httpjson.yml.hbs

@@ -16,6 +16,9 @@ auth.oauth2.scopes:
 auth.oauth2.provider: azure
 auth.oauth2.azure.resource: {{azure_resource}}
 {{/if}}
+{{#if oauth_endpoint_params}}
+auth.oauth2.endpoint_params: {{oauth_endpoint_params}}
+{{/if}}
 request.url: {{request_url}}
 request.method: GET
 {{#if proxy_url }}

packages/microsoft_defender_endpoint/data_stream/log/manifest.yml

@@ -94,6 +94,13 @@ streams:
         default: oauth2/token
         description: "Microsoft supports multiple Oauth2 URL endpoints, the default is oauth2/token, but can also be oauth2/v2.0/token"
         secret: false
+      - name: oauth_endpoint_params
+        type: yaml
+        title: OAuth2 Endpoint Params
+        description: Endpoint Params used for OAuth2 authentication as YAML. See [documentation](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-httpjson#_auth_oauth2_endpoint_params_2) for details.
+        show_user: false
+        multi: false
+        required: false
       - name: request_url
         type: text
         title: Security Center URL

packages/microsoft_defender_endpoint/data_stream/log/sample_event.json

@@ -1,101 +1,97 @@
 {
-    "@timestamp": "2025-09-08T14:14:17.520Z",
+    "@timestamp": "2025-10-16T10:42:06.592Z",
     "agent": {
-        "ephemeral_id": "3f082892-0f6c-45a1-aa4d-4887bd3462c8",
-        "id": "36f3bbaf-2d4f-4f3c-af1f-4c13524514f5",
-        "name": "elastic-agent-19107",
+        "ephemeral_id": "b43b835d-ba25-4ad7-aa9f-200c15733e48",
+        "id": "353e6432-27a9-40a4-98a2-557dbb69496b",
+        "name": "elastic-agent-48674",
         "type": "filebeat",
-        "version": "8.19.0"
+        "version": "8.19.3"
     },
     "cloud": {
         "account": {
-            "id": "123543-d66c-4c7e-9e30-40034eb7c6f3"
+            "id": "a839b112-1253-6432-9bf6-94542403f21c"
         },
         "instance": {
-            "id": "c5a964f417c11f6277d5bf9489f0d"
+            "id": "111e6dd8c833c8a052ea231ec1b19adaf497b625"
         },
         "provider": "azure"
     },
     "data_stream": {
         "dataset": "microsoft_defender_endpoint.log",
-        "namespace": "61125",
+        "namespace": "90146",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "36f3bbaf-2d4f-4f3c-af1f-4c13524514f5",
+        "id": "353e6432-27a9-40a4-98a2-557dbb69496b",
         "snapshot": false,
-        "version": "8.19.0"
+        "version": "8.19.3"
     },
     "event": {
-        "action": "Malware",
+        "action": "Execution",
         "agent_id_status": "verified",
         "category": [
-            "host",
-            "malware"
+            "host"
         ],
-        "created": "2020-06-30T10:09:01.1569718Z",
+        "created": "2021-01-26T20:33:57.7220239Z",
         "dataset": "microsoft_defender_endpoint.log",
-        "duration": 0,
-        "end": "2020-06-30T10:07:44.333733Z",
-        "id": "da637291085411733957_-1043898914",
-        "ingested": "2025-09-08T14:14:20Z",
+        "duration": 101466100,
+        "end": "2021-01-26T20:31:33.0577322Z",
+        "id": "da637472900382838869_1364969609",
+        "ingested": "2025-10-16T10:42:09Z",
         "kind": "alert",
         "provider": "defender_endpoint",
         "severity": 21,
-        "start": "2020-06-30T10:07:44.333733Z",
+        "start": "2021-01-26T20:31:32.9562661Z",
         "timezone": "UTC",
         "type": [
-            "end"
+            "access",
+            "start"
         ]
     },
-    "file": {
-        "name": "SB.xsl",
-        "path": "C:\\Windows\\Temp\\sb-sim-temp-ikyxqi\\sb_10554_bs_h4qpk5"
-    },
     "host": {
-        "hostname": "testserver4",
-        "id": "c5a964f417c11f6277d5bf9489f0d",
-        "name": "testserver4"
+        "hostname": "temp123.middleeast.corp.microsoft.com",
+        "id": "111e6dd8c833c8a052ea231ec1b19adaf497b625",
+        "name": "temp123.middleeast.corp.microsoft.com"
     },
     "input": {
-        "type": "log"
-    },
-    "log": {
-        "file": {
-            "path": "/tmp/service_logs/defender_atp-test.json.log"
-        },
-        "offset": 0
+        "type": "httpjson"
     },
-    "message": "An active 'Exeselrun' malware was detected",
+    "message": "Low-reputation arbitrary code executed by signed executable",
     "microsoft": {
         "defender_endpoint": {
             "assignedTo": "[email protected]",
             "evidence": {
-                "entityType": "File"
+                "aadUserId": "11118379-2a59-1111-ac3c-a51eb4a3c627",
+                "accountName": "name",
+                "domainName": "DOMAIN",
+                "entityType": "User",
+                "userPrincipalName": "[email protected]"
             },
-            "incidentId": "12",
-            "investigationId": "9",
-            "investigationState": "Benign",
-            "lastUpdateTime": "2020-07-03T15:15:39.13Z",
-            "resolvedTime": "2020-06-30T11:13:12.2680434Z",
-            "status": "Resolved"
+            "incidentId": "1126093",
+            "investigationState": "Queued",
+            "lastUpdateTime": "2021-01-26T20:33:59.2Z",
+            "rbacGroupName": "A",
+            "status": "New"
         }
     },
     "observer": {
-        "name": "WindowsDefenderAv",
+        "name": "WindowsDefenderAtp",
         "product": "Defender for Endpoint",
         "vendor": "Microsoft"
     },
     "related": {
         "hosts": [
-            "testserver4"
+            "temp123.middleeast.corp.microsoft.com"
+        ],
+        "user": [
+            "temp123"
         ]
     },
     "rule": {
-        "description": "Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection."
+        "description": "Binaries signed by Microsoft can be used to run low-reputation arbitrary code. This technique hides the execution of malicious code within a trusted process. As a result, the trusted process might exhibit suspicious behaviors, such as opening a listening port or connecting to a command-and-control (C&C) server."
     },
     "tags": [
         "microsoft-defender-endpoint",
@@ -105,8 +101,13 @@
         "framework": "MITRE ATT&CK",
         "technique": {
             "name": [
-                "Malware"
+                "Execution"
             ]
         }
+    },
+    "user": {
+        "domain": "DOMAIN",
+        "id": "S-1-5-21-11111607-1111760036-109187956-75141",
+        "name": "temp123"
     }
 }

packages/microsoft_defender_endpoint/data_stream/machine/_dev/test/system/test-oauth-params-config.yml

@@ -0,0 +1,18 @@
+input: cel
+service: microsoft-defender-mock
+vars:
+  url: http://{{Hostname}}:{{Port}}
+  login_url: http://{{Hostname}}:{{Port}}
+  client_id: test-cel-client-id
+  client_secret: test-cel-client-secret
+  tenant_id: tenant-id
+  oauth_endpoint_params: |
+      grant_type: refresh_token
+      refresh_token: 'refresh_token_123'
+data_stream:
+  vars:
+    preserve_original_event: true
+    enable_request_tracer: true
+    batch_size: 2
+assert:
+  hit_count: 5

packages/microsoft_defender_endpoint/data_stream/machine/agent/stream/cel.yml.hbs

@@ -16,6 +16,9 @@ auth.oauth2:
   client.secret: {{client_secret}}
   token_url:  {{login_url}}/{{tenant_id}}/oauth2/v2.0/token
   scopes: {{token_scopes}}
+{{#if oauth_endpoint_params}}
+  endpoint_params: {{oauth_endpoint_params}}
+{{/if}}
 state:
   page_size: {{batch_size}}
 redact: