diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml index c2cdcbebcc5..3078f891ffa 100644 --- a/packages/azure/changelog.yml +++ b/packages/azure/changelog.yml @@ -1,3 +1,8 @@ +- version: "1.29.0" + changes: + - description: Parse responseBody and requestBody json in activitylogs. + type: enhancement + link: https://github.com/elastic/integrations/pull/15690 - version: "1.28.7" changes: - description: Interim fix to support non-standard log events. diff --git a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log index d5c8c1930cd..68f81ad33f2 100644 --- a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log +++ b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log @@ -1 +1,3 @@ -{"callerIpAddress":"81.2.69.144","category":"Action","correlationId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","durationMs":0,"identity":{"authorization":{"action":"Microsoft.EventHub/namespaces/authorizationRules/listKeys/action","evidence":{"principalId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","principalType":"ServicePrincipal","role":"Azure EventGrid Service BuiltIn Role","roleAssignmentId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","roleAssignmentScope":"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53","roleDefinitionId":"8a4de8b5-095c-47d0-a96f-a75130c61d53"},"scope":"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53/resourceGroups/sa-hem/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey"},"claims":{"aio":"8a4de8b5-095c-47d0-a96f-a75130c61d53","appid":"8a4de8b5-095c-47d0-a96f-a75130c61d53","appidacr":"2","aud":"https://management.core.windows.net/","exp":"1571904826","http://schemas.microsoft.com/identity/claims/identityprovider":"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/","http://schemas.microsoft.com/identity/claims/objectidentifier":"8a4de8b5-095c-47d0-a96f-a75130c61d53","http://schemas.microsoft.com/identity/claims/tenantid":"8a4de8b5-095c-47d0-a96f-a75130c61d53","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier":"8a4de8b5-095c-47d0-a96f-a75130c61d53","iat":"1571875726","iss":"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/","nbf":"1571875726","uti":"8a4de8b5-095c-47d0-a96f-a75130c61d53","ver":"1.0"}},"level":"Information","location":"global","operationName":"MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION","resourceId":"/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY","resultSignature":"Started.","resultType":"Start","time":"2019-10-24T00:13:46.3554259Z"} \ No newline at end of file +{"callerIpAddress":"81.2.69.144","category":"Action","correlationId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","durationMs":0,"identity":{"authorization":{"action":"Microsoft.EventHub/namespaces/authorizationRules/listKeys/action","evidence":{"principalId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","principalType":"ServicePrincipal","role":"Azure EventGrid Service BuiltIn Role","roleAssignmentId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","roleAssignmentScope":"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53","roleDefinitionId":"8a4de8b5-095c-47d0-a96f-a75130c61d53"},"scope":"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53/resourceGroups/sa-hem/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey"},"claims":{"aio":"8a4de8b5-095c-47d0-a96f-a75130c61d53","appid":"8a4de8b5-095c-47d0-a96f-a75130c61d53","appidacr":"2","aud":"https://management.core.windows.net/","exp":"1571904826","http://schemas.microsoft.com/identity/claims/identityprovider":"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/","http://schemas.microsoft.com/identity/claims/objectidentifier":"8a4de8b5-095c-47d0-a96f-a75130c61d53","http://schemas.microsoft.com/identity/claims/tenantid":"8a4de8b5-095c-47d0-a96f-a75130c61d53","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier":"8a4de8b5-095c-47d0-a96f-a75130c61d53","iat":"1571875726","iss":"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/","nbf":"1571875726","uti":"8a4de8b5-095c-47d0-a96f-a75130c61d53","ver":"1.0"}},"level":"Information","location":"global","operationName":"MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION","resourceId":"/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY","resultSignature":"Started.","resultType":"Start","time":"2019-10-24T00:13:46.3554259Z"} +{"category":"ResourceHealth","correlationId":"1c867fe2-050c-4a74-bb1c-a83b15246fdd","level":"Information","operationName":"Microsoft.Resourcehealth/healthevent/Updated/action","properties":{"responseBody": "{\"skuTest\":{\"myName\":\"Standard_LRS\"}}", "eventCategory":"ResourceHealth","eventProperties":{"cause":"PlatformInitiated"}},"resourceId":"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration","resultType":"Updated","time":"2025-10-17T11:50:07.22Z"} +{"category":"ResourceHealth","correlationId":"1c867fe2-050c-4a74-bb1c-a83b15246fdd","level":"Information","operationName":"Microsoft.Resourcehealth/healthevent/Updated/action","properties":{"responseBody": {"skuTest":{"myName":"Standard_LRS"}}, "eventCategory":"ResourceHealth","eventProperties":{"cause":"PlatformInitiated"}},"resourceId":"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration","resultType":"Updated","time":"2025-10-17T11:50:07.22Z"} \ No newline at end of file diff --git a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json index 685772b74f4..a56dc5b644b 100644 --- a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json +++ b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json @@ -110,6 +110,104 @@ "tags": [ "preserve_original_event" ] + }, + { + "@timestamp": "2025-10-17T11:50:07.220Z", + "azure": { + "activitylogs": { + "category": "ResourceHealth", + "event_category": "ResourceHealth", + "operation_name": "Microsoft.Resourcehealth/healthevent/Updated/action", + "properties": { + "eventProperties": { + "cause": "PlatformInitiated" + }, + "response_body": { + "sku_test": { + "my_name": "Standard_LRS" + } + } + }, + "result_type": "Updated" + }, + "correlation_id": "1c867fe2-050c-4a74-bb1c-a83b15246fdd", + "resource": { + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration", + "provider": "Microsoft.domainRegistration" + }, + "subscription_id": "00000000-0000-0000-0000-000000000000" + }, + "cloud": { + "provider": "azure" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "Microsoft.Resourcehealth/healthevent/Updated/action", + "kind": "event", + "original": "{\"category\":\"ResourceHealth\",\"correlationId\":\"1c867fe2-050c-4a74-bb1c-a83b15246fdd\",\"level\":\"Information\",\"operationName\":\"Microsoft.Resourcehealth/healthevent/Updated/action\",\"properties\":{\"responseBody\": \"{\\\"skuTest\\\":{\\\"myName\\\":\\\"Standard_LRS\\\"}}\", \"eventCategory\":\"ResourceHealth\",\"eventProperties\":{\"cause\":\"PlatformInitiated\"}},\"resourceId\":\"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration\",\"resultType\":\"Updated\",\"time\":\"2025-10-17T11:50:07.22Z\"}" + }, + "log": { + "level": "Information" + }, + "related": { + "entity": [ + "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration" + ] + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2025-10-17T11:50:07.220Z", + "azure": { + "activitylogs": { + "category": "ResourceHealth", + "event_category": "ResourceHealth", + "operation_name": "Microsoft.Resourcehealth/healthevent/Updated/action", + "properties": { + "eventProperties": { + "cause": "PlatformInitiated" + }, + "response_body": { + "sku_test": { + "my_name": "Standard_LRS" + } + } + }, + "result_type": "Updated" + }, + "correlation_id": "1c867fe2-050c-4a74-bb1c-a83b15246fdd", + "resource": { + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration", + "provider": "Microsoft.domainRegistration" + }, + "subscription_id": "00000000-0000-0000-0000-000000000000" + }, + "cloud": { + "provider": "azure" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "Microsoft.Resourcehealth/healthevent/Updated/action", + "kind": "event", + "original": "{\"category\":\"ResourceHealth\",\"correlationId\":\"1c867fe2-050c-4a74-bb1c-a83b15246fdd\",\"level\":\"Information\",\"operationName\":\"Microsoft.Resourcehealth/healthevent/Updated/action\",\"properties\":{\"responseBody\": {\"skuTest\":{\"myName\":\"Standard_LRS\"}}, \"eventCategory\":\"ResourceHealth\",\"eventProperties\":{\"cause\":\"PlatformInitiated\"}},\"resourceId\":\"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration\",\"resultType\":\"Updated\",\"time\":\"2025-10-17T11:50:07.22Z\"}" + }, + "log": { + "level": "Information" + }, + "related": { + "entity": [ + "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration" + ] + }, + "tags": [ + "preserve_original_event" + ] } ] -} \ No newline at end of file +} diff --git a/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml index 85b1c26dc8b..ac63b29e717 100644 --- a/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml @@ -96,6 +96,46 @@ processors: field: azure.activitylogs.properties if: "ctx.azure?.activitylogs?.properties instanceof String" ignore_failure: true + - json: + field: azure.activitylogs.properties.responseBody + if: "ctx.azure?.activitylogs?.properties?.responseBody instanceof String" + ignore_failure: true + - json: + field: azure.activitylogs.properties.requestBody + if: "ctx.azure?.activitylogs?.properties?.requestBody instanceof String" + ignore_failure: true + - script: + lang: painless + source: >- + Map toSnakeCase(Map obj) { + for (def camelKey : new ArrayList(obj.keySet())) { + StringBuilder snakeKeyBuilder = new StringBuilder(); + for (char c : camelKey.toCharArray()) { + if (Character.isUpperCase(c)) { + snakeKeyBuilder.append('_'); + } + snakeKeyBuilder.append(Character.toLowerCase(c)); + } + def snakeKey = snakeKeyBuilder.toString(); + + if (!camelKey.equals(snakeKey)) { + obj[snakeKey] = obj.remove(camelKey); + } + + if (obj[snakeKey] instanceof Map) { + obj[snakeKey] = toSnakeCase(obj[snakeKey]) + } + } + + return obj; + } + if (ctx?.azure?.activitylogs?.properties?.responseBody instanceof Map) { + toSnakeCase(ctx?.azure?.activitylogs?.properties?.responseBody) + } + if (ctx?.azure?.activitylogs?.properties?.requestBody instanceof Map) { + toSnakeCase(ctx?.azure?.activitylogs?.properties?.requestBody) + } + ignore_failure: true - script: lang: painless source: >- @@ -109,6 +149,14 @@ processors: ctx.azure.activitylogs.event_category = 'Administrative'; } ignore_failure: true + - rename: + field: azure.activitylogs.properties.responseBody + target_field: azure.activitylogs.properties.response_body + ignore_missing: true + - rename: + field: azure.activitylogs.properties.requestBody + target_field: azure.activitylogs.properties.request_body + ignore_missing: true - remove: field: azure.activitylogs.properties.eventCategory if: 'ctx.azure.activitylogs.event_category != null' diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml index 892f9aea996..73345378ce8 100644 --- a/packages/azure/manifest.yml +++ b/packages/azure/manifest.yml @@ -1,6 +1,6 @@ name: azure title: Azure Logs -version: "1.28.7" +version: "1.29.0" description: This Elastic integration collects logs from Azure type: integration icons: