[azure][activitylogs] add json processor to responseBody and requestBody

packages/azure/changelog.yml

@@ -1,3 +1,8 @@
+- version: "1.29.0"
+  changes:
+    - description: Parse responseBody and requestBody json in activitylogs.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15690
 - version: "1.28.7"
   changes:
     - description: Interim fix to support non-standard log events.

packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log

@@ -1 +1,3 @@
-{"callerIpAddress":"81.2.69.144","category":"Action","correlationId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","durationMs":0,"identity":{"authorization":{"action":"Microsoft.EventHub/namespaces/authorizationRules/listKeys/action","evidence":{"principalId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","principalType":"ServicePrincipal","role":"Azure EventGrid Service BuiltIn Role","roleAssignmentId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","roleAssignmentScope":"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53","roleDefinitionId":"8a4de8b5-095c-47d0-a96f-a75130c61d53"},"scope":"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53/resourceGroups/sa-hem/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey"},"claims":{"aio":"8a4de8b5-095c-47d0-a96f-a75130c61d53","appid":"8a4de8b5-095c-47d0-a96f-a75130c61d53","appidacr":"2","aud":"https://management.core.windows.net/","exp":"1571904826","http://schemas.microsoft.com/identity/claims/identityprovider":"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/","http://schemas.microsoft.com/identity/claims/objectidentifier":"8a4de8b5-095c-47d0-a96f-a75130c61d53","http://schemas.microsoft.com/identity/claims/tenantid":"8a4de8b5-095c-47d0-a96f-a75130c61d53","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier":"8a4de8b5-095c-47d0-a96f-a75130c61d53","iat":"1571875726","iss":"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/","nbf":"1571875726","uti":"8a4de8b5-095c-47d0-a96f-a75130c61d53","ver":"1.0"}},"level":"Information","location":"global","operationName":"MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION","resourceId":"/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY","resultSignature":"Started.","resultType":"Start","time":"2019-10-24T00:13:46.3554259Z"}
+{"callerIpAddress":"81.2.69.144","category":"Action","correlationId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","durationMs":0,"identity":{"authorization":{"action":"Microsoft.EventHub/namespaces/authorizationRules/listKeys/action","evidence":{"principalId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","principalType":"ServicePrincipal","role":"Azure EventGrid Service BuiltIn Role","roleAssignmentId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","roleAssignmentScope":"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53","roleDefinitionId":"8a4de8b5-095c-47d0-a96f-a75130c61d53"},"scope":"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53/resourceGroups/sa-hem/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey"},"claims":{"aio":"8a4de8b5-095c-47d0-a96f-a75130c61d53","appid":"8a4de8b5-095c-47d0-a96f-a75130c61d53","appidacr":"2","aud":"https://management.core.windows.net/","exp":"1571904826","http://schemas.microsoft.com/identity/claims/identityprovider":"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/","http://schemas.microsoft.com/identity/claims/objectidentifier":"8a4de8b5-095c-47d0-a96f-a75130c61d53","http://schemas.microsoft.com/identity/claims/tenantid":"8a4de8b5-095c-47d0-a96f-a75130c61d53","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier":"8a4de8b5-095c-47d0-a96f-a75130c61d53","iat":"1571875726","iss":"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/","nbf":"1571875726","uti":"8a4de8b5-095c-47d0-a96f-a75130c61d53","ver":"1.0"}},"level":"Information","location":"global","operationName":"MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION","resourceId":"/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY","resultSignature":"Started.","resultType":"Start","time":"2019-10-24T00:13:46.3554259Z"}
+{"category":"ResourceHealth","correlationId":"1c867fe2-050c-4a74-bb1c-a83b15246fdd","level":"Information","operationName":"Microsoft.Resourcehealth/healthevent/Updated/action","properties":{"responseBody": "{\"skuTest\":{\"myName\":\"Standard_LRS\"}}", "eventCategory":"ResourceHealth","eventProperties":{"cause":"PlatformInitiated"}},"resourceId":"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration","resultType":"Updated","time":"2025-10-17T11:50:07.22Z"}
+{"category":"ResourceHealth","correlationId":"1c867fe2-050c-4a74-bb1c-a83b15246fdd","level":"Information","operationName":"Microsoft.Resourcehealth/healthevent/Updated/action","properties":{"responseBody": {"skuTest":{"myName":"Standard_LRS"}}, "eventCategory":"ResourceHealth","eventProperties":{"cause":"PlatformInitiated"}},"resourceId":"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration","resultType":"Updated","time":"2025-10-17T11:50:07.22Z"}

packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json

@@ -110,6 +110,104 @@
             "tags": [
                 "preserve_original_event"
             ]
+        },
+        {
+            "@timestamp": "2025-10-17T11:50:07.220Z",
+            "azure": {
+                "activitylogs": {
+                    "category": "ResourceHealth",
+                    "event_category": "ResourceHealth",
+                    "operation_name": "Microsoft.Resourcehealth/healthevent/Updated/action",
+                    "properties": {
+                        "eventProperties": {
+                            "cause": "PlatformInitiated"
+                        },
+                        "response_body": {
+                            "sku_test": {
+                                "my_name": "Standard_LRS"
+                            }
+                        }
+                    },
+                    "result_type": "Updated"
+                },
+                "correlation_id": "1c867fe2-050c-4a74-bb1c-a83b15246fdd",
+                "resource": {
+                    "id": "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration",
+                    "provider": "Microsoft.domainRegistration"
+                },
+                "subscription_id": "00000000-0000-0000-0000-000000000000"
+            },
+            "cloud": {
+                "provider": "azure"
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "action": "Microsoft.Resourcehealth/healthevent/Updated/action",
+                "kind": "event",
+                "original": "{\"category\":\"ResourceHealth\",\"correlationId\":\"1c867fe2-050c-4a74-bb1c-a83b15246fdd\",\"level\":\"Information\",\"operationName\":\"Microsoft.Resourcehealth/healthevent/Updated/action\",\"properties\":{\"responseBody\": \"{\\\"skuTest\\\":{\\\"myName\\\":\\\"Standard_LRS\\\"}}\", \"eventCategory\":\"ResourceHealth\",\"eventProperties\":{\"cause\":\"PlatformInitiated\"}},\"resourceId\":\"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration\",\"resultType\":\"Updated\",\"time\":\"2025-10-17T11:50:07.22Z\"}"
+            },
+            "log": {
+                "level": "Information"
+            },
+            "related": {
+                "entity": [
+                    "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
+        },
+        {
+            "@timestamp": "2025-10-17T11:50:07.220Z",
+            "azure": {
+                "activitylogs": {
+                    "category": "ResourceHealth",
+                    "event_category": "ResourceHealth",
+                    "operation_name": "Microsoft.Resourcehealth/healthevent/Updated/action",
+                    "properties": {
+                        "eventProperties": {
+                            "cause": "PlatformInitiated"
+                        },
+                        "response_body": {
+                            "sku_test": {
+                                "my_name": "Standard_LRS"
+                            }
+                        }
+                    },
+                    "result_type": "Updated"
+                },
+                "correlation_id": "1c867fe2-050c-4a74-bb1c-a83b15246fdd",
+                "resource": {
+                    "id": "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration",
+                    "provider": "Microsoft.domainRegistration"
+                },
+                "subscription_id": "00000000-0000-0000-0000-000000000000"
+            },
+            "cloud": {
+                "provider": "azure"
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "action": "Microsoft.Resourcehealth/healthevent/Updated/action",
+                "kind": "event",
+                "original": "{\"category\":\"ResourceHealth\",\"correlationId\":\"1c867fe2-050c-4a74-bb1c-a83b15246fdd\",\"level\":\"Information\",\"operationName\":\"Microsoft.Resourcehealth/healthevent/Updated/action\",\"properties\":{\"responseBody\": {\"skuTest\":{\"myName\":\"Standard_LRS\"}}, \"eventCategory\":\"ResourceHealth\",\"eventProperties\":{\"cause\":\"PlatformInitiated\"}},\"resourceId\":\"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration\",\"resultType\":\"Updated\",\"time\":\"2025-10-17T11:50:07.22Z\"}"
+            },
+            "log": {
+                "level": "Information"
+            },
+            "related": {
+                "entity": [
+                    "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         }
     ]
-}
+}

packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml

@@ -96,6 +96,46 @@ processors:
       field: azure.activitylogs.properties
       if: "ctx.azure?.activitylogs?.properties instanceof String"
       ignore_failure: true
+  - json:
+      field: azure.activitylogs.properties.responseBody
+      if: "ctx.azure?.activitylogs?.properties?.responseBody instanceof String"
+      ignore_failure: true
+  - json:
+      field: azure.activitylogs.properties.requestBody
+      if: "ctx.azure?.activitylogs?.properties?.requestBody instanceof String"
+      ignore_failure: true
+  - script:
+      lang: painless
+      source: >-
+        Map toSnakeCase(Map obj) {
+          for (def camelKey : new ArrayList(obj.keySet())) {
+            StringBuilder snakeKeyBuilder = new StringBuilder();
+            for (char c : camelKey.toCharArray()) {
+              if (Character.isUpperCase(c)) {
+                snakeKeyBuilder.append('_');
+              } 
+              snakeKeyBuilder.append(Character.toLowerCase(c));
+            }
+            def snakeKey = snakeKeyBuilder.toString();
+
+            if (!camelKey.equals(snakeKey)) {
+              obj[snakeKey] = obj.remove(camelKey);
+            }
+
+            if (obj[snakeKey] instanceof Map) {
+              obj[snakeKey] = toSnakeCase(obj[snakeKey])
+            }
+          }
+
+          return obj;
+        }
+        if (ctx?.azure?.activitylogs?.properties?.responseBody instanceof Map) {
+          toSnakeCase(ctx?.azure?.activitylogs?.properties?.responseBody)
+        }
+        if (ctx?.azure?.activitylogs?.properties?.requestBody instanceof Map) {
+          toSnakeCase(ctx?.azure?.activitylogs?.properties?.requestBody)
+        }
+      ignore_failure: true
   - script:
       lang: painless
       source: >-
@@ -109,6 +149,14 @@ processors:
           ctx.azure.activitylogs.event_category = 'Administrative';
         }
       ignore_failure: true
+  - rename:
+      field: azure.activitylogs.properties.responseBody
+      target_field: azure.activitylogs.properties.response_body
+      ignore_missing: true
+  - rename:
+      field: azure.activitylogs.properties.requestBody
+      target_field: azure.activitylogs.properties.request_body
+      ignore_missing: true
   - remove:
       field: azure.activitylogs.properties.eventCategory
       if: 'ctx.azure.activitylogs.event_category != null'

packages/azure/manifest.yml

@@ -1,6 +1,6 @@
 name: azure
 title: Azure Logs
-version: "1.28.7"
+version: "1.29.0"
 description: This Elastic integration collects logs from Azure
 type: integration
 icons: