From 258fddd03150d9647878089b643fef3e83b23ec4 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 18 Nov 2024 11:56:00 +0100 Subject: [PATCH 1/6] document related.entity field --- packages/gcp/data_stream/audit/fields/fields.yml | 7 +++++++ packages/gcp/docs/audit.md | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/gcp/data_stream/audit/fields/fields.yml b/packages/gcp/data_stream/audit/fields/fields.yml index ee4885c98eb..19a30dc6a55 100644 --- a/packages/gcp/data_stream/audit/fields/fields.yml +++ b/packages/gcp/data_stream/audit/fields/fields.yml @@ -113,6 +113,13 @@ - name: policyType type: keyword description: "Indicates the type of the policy." + - name: related.entity + description: | + A collection of all entity identifiers associated with the document. + If the document contains multiple entities, identifiers for each will be included. + Example identifiers include (but not limited to) cloud resource IDs, email addresses, + and hostnames. + type: keyword - name: request type: flattened - name: request_metadata diff --git a/packages/gcp/docs/audit.md b/packages/gcp/docs/audit.md index 720b5b494e7..44faed19057 100644 --- a/packages/gcp/docs/audit.md +++ b/packages/gcp/docs/audit.md @@ -77,7 +77,7 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | - +| related.entity | A collection of all entity identifiers associated with the document. If the document contains multiple entities, identifiers for each will be included. Example identifiers include (but not limited to) cloud resource IDs, email addresses, and hostnames. | keyword | An example event for `audit` looks as following: @@ -203,4 +203,4 @@ An example event for `audit` looks as following: "version": "71.0." } } -``` \ No newline at end of file +``` From 88a3007cfb3a5ce60f5a7e998dcf85e54745095d Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 18 Nov 2024 11:56:57 +0100 Subject: [PATCH 2/6] add related.entity to audit logs and test it --- packages/gcp/changelog.yml | 5 + .../pipeline/test-audit.log-expected.json | 78 +++++- ...st-cloudresourcemanager-googleapis-com.log | 1 + ...cemanager-googleapis-com.log-expected.json | 198 ++++++++++++++ .../pipeline/test-compute-googleapis-com.log | 1 + ...t-compute-googleapis-com.log-expected.json | 254 ++++++++++++++++++ .../test-iamcredentials-googleapis-com.log | 1 + ...edentials-googleapis-com.log-expected.json | 105 ++++++++ .../pipeline/test-sdh-3695.log-expected.json | 6 +- .../elasticsearch/ingest_pipeline/default.yml | 118 +++++++- .../data_stream/audit/fields/base-fields.yml | 7 + .../gcp/data_stream/audit/fields/fields.yml | 7 - packages/gcp/docs/README.md | 1 + packages/gcp/docs/audit.md | 3 +- packages/gcp/manifest.yml | 2 +- 15 files changed, 768 insertions(+), 19 deletions(-) create mode 100644 packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log create mode 100644 packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log-expected.json create mode 100644 packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log create mode 100644 packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log-expected.json create mode 100644 packages/gcp/data_stream/audit/_dev/test/pipeline/test-iamcredentials-googleapis-com.log create mode 100644 packages/gcp/data_stream/audit/_dev/test/pipeline/test-iamcredentials-googleapis-com.log-expected.json diff --git a/packages/gcp/changelog.yml b/packages/gcp/changelog.yml index 415fb19aeac..696bd94890f 100644 --- a/packages/gcp/changelog.yml +++ b/packages/gcp/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.39.0" + changes: + - description: WIP KUBA DEBUG - Adding related.entity + type: enhancement + link: TBD - version: "2.38.0" changes: - description: Add `policy_violation_info`, `metadata` and `related` fields to audit logs. diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json index 8ea7905c637..dc2cb2a31f5 100644 --- a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json @@ -59,6 +59,10 @@ ], "user": [ "xxx@xxx.xxx" + ], + "entity": [ + "projects/elastic-beats", + "xxx@xxx.xxx" ] }, "service": { @@ -139,6 +143,10 @@ ], "user": [ "xxx@xxx.xxx" + ], + "entity": [ + "projects/elastic-beats/global/machineTypes", + "xxx@xxx.xxx" ] }, "service": { @@ -244,6 +252,10 @@ ], "user": [ "xxx@xxx.xxx" + ], + "entity": [ + "projects/elastic-beats/global/instances", + "xxx@xxx.xxx" ] }, "service": { @@ -336,6 +348,10 @@ ], "user": [ "xxx@xxx.xxx" + ], + "entity": [ + "projects/elastic-beats/global/instances", + "xxx@xxx.xxx" ] }, "service": { @@ -475,7 +491,8 @@ ], "user": [ "system:serviceaccount:cert-manager:cert-manager-webhook" - ] + ], + "entity": [] }, "service": { "name": "k8s.io" @@ -598,6 +615,10 @@ ], "user": [ "user@mycompany.com" + ], + "entity": [ + "projects/foo/global/images/windows-server-2016-v20200805", + "user@mycompany.com" ] }, "service": { @@ -689,6 +710,10 @@ ], "user": [ "user@mycompany.com" + ], + "entity": [ + "projects/foo/zones/us-central1-a/instances/win10-test", + "user@mycompany.com" ] }, "service": { @@ -792,7 +817,8 @@ ], "user": [ "xxx@xxx.xxx" - ] + ], + "entity": [] }, "service": { "name": "k8s.io" @@ -880,7 +906,8 @@ ], "user": [ "xxx@xxx.xxx" - ] + ], + "entity": [] }, "service": { "name": "k8s.io" @@ -965,7 +992,8 @@ ], "user": [ "system:anonymous" - ] + ], + "entity": [] }, "service": { "name": "k8s.io" @@ -1048,7 +1076,8 @@ ], "user": [ "system:serviceaccount:kube-system:generic-garbage-collector" - ] + ], + "entity": [] }, "service": { "name": "k8s.io" @@ -1131,6 +1160,12 @@ "related": { "user": [ "xxx@xxx.xxx" + ], + "entity": [ + "projects/project", + "sub", + "xxx@xxx.xxx", + "//xxx@xxx" ] }, "service": { @@ -1266,6 +1301,7 @@ "type": "kubernetes" }, "related": { + "entity": [], "ip": [ "67.43.156.13" ], @@ -1656,6 +1692,7 @@ "type": "kubernetes" }, "related": { + "entity": [], "ip": [ "10.142.0.152" ], @@ -1747,6 +1784,9 @@ "type": "kubernetes" }, "related": { + "entity": [ + "serviceAccount:service-xxxx@developer.gserviceaccount.com" + ], "ip": [ "192.168.1.1" ], @@ -1826,6 +1866,10 @@ "logger": "projects/elastic/logs/cloudaudit.googleapis.com%2Fdata_access" }, "related": { + "entity": [ + "projects/_/buckets/dataflow-staging-us-central1-xxx/objects/staging/jfxrt-xxx.jar", + "xxx-compute@developer.gserviceaccount.com" + ], "user": [ "xxx-compute@developer.gserviceaccount.com" ] @@ -1909,6 +1953,9 @@ "type": "kubernetes" }, "related": { + "entity": [ + "serviceAccount:servoce-xxxx@developer.gserviceaccount.com" + ], "ip": [ "192.168.1.1" ], @@ -1992,6 +2039,12 @@ "logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access" }, "related": { + "entity": [ + "projects/project", + "sub", + "xxx@xxx.xxx", + "//xxx@xxx" + ], "user": [ "xxx@xxx.xxx" ] @@ -2060,6 +2113,10 @@ "logger": "projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fsystem_event" }, "related": { + "entity": [ + "projects/elastic-siem/zones/us-central1-c/instances/sep-perf-debian-11-155", + "system@google.com" + ], "user": [ "system@google.com" ] @@ -2138,6 +2195,9 @@ "logger": "projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fpolicy" }, "related": { + "entity": [ + "projects/elastic-siem" + ], "ip": [ "192.168.1.1" ] @@ -2236,6 +2296,9 @@ "type": "kubernetes" }, "related": { + "entity": [ + "serviceAccount:servoce-xxxx@developer.gserviceaccount.com" + ], "ip": [ "192.168.1.1" ], @@ -2311,6 +2374,9 @@ }, "type": "kubernetes" }, + "related": { + "entity": [] + }, "service": { "name": "container.googleapis.com" }, @@ -2319,4 +2385,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log new file mode 100644 index 00000000000..aef09eca3f0 --- /dev/null +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log @@ -0,0 +1 @@ +{"insertId":"-30102re2sad8","logName":"projects/project-id/logs/cloudaudit.googleapis.com%2Factivity","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"made-up-ci-account@project-id.iam.gserviceaccount.com","principalSubject":"serviceAccount:madeupprincipal@project-id.iam.gserviceaccount.com","serviceAccountDelegationInfo":[{"principalSubject":"principal://iam.googleapis.com/projects/project-id/locations/global/workloadIdentityPools/..."}]},"authorizationInfo":[{"granted":true,"permission":"resourcemanager.projects.setIamPolicy","permissionType":"ADMIN_WRITE","resource":"projects/project-id","resourceAttributes":{"name":"projects/project-id","service":"cloudresourcemanager.googleapis.com","type":"cloudresourcemanager.googleapis.com/Project"}},{"granted":true,"permission":"resourcemanager.projects.setIamPolicy","permissionType":"ADMIN_WRITE","resource":"projects/project-id","resourceAttributes":{"name":"projects/project-id","service":"cloudresourcemanager.googleapis.com","type":"cloudresourcemanager.googleapis.com/Project"}}],"methodName":"SetIamPolicy","request":{"@type":"type.googleapis.com/google.iam.v1.SetIamPolicyRequest","policy":{"bindings":[{"members":["serviceAccount:member-sa@project-id.iam.gserviceaccount.com"],"role":"projects/project-id/roles/ThatRoleToo"},{"members":["serviceAccount:a@project-id.iam.gserviceaccount.com"],"role":"projects/project-id/roles/x"},{"members":["serviceAccount:b@project-id.iam.gserviceaccount.com"],"role":"projects/project-id/roles/this_role_as_well"},{"members":["serviceAccount:c@project-id.iam.gserviceaccount.com","serviceAccount:d@project-id.iam.gserviceaccount.com","serviceAccount:e@project-id.iam.gserviceaccount.com"],"role":"roles/browser"},{"members":["serviceAccount:f@project-id.iam.gserviceaccount.com","serviceAccount:g@project-id.iam.gserviceaccount.com","serviceAccount:c@project-id.iam.gserviceaccount.com"],"role":"roles/cloudasset.viewer"},{"members":["user:doesnotexist@elastic.co"],"role":"roles/cloudkms.admin"},{"members":["group:agroup@elastic.co"],"role":"roles/owner"}],"etag":"BwYnObHBOBA="},"resource":"project-id"},"requestMetadata":{"callerIp":"192.168.0.1","callerSuppliedUserAgent":"google-cloud-sdk gcloud/501.0.0 command/gcloud.projects.add-iam-policy-binding invocation-id/e9e9e4b6f9294a7da9a2247dc101225a environment/None environment-version/None client-os/LINUX client-os-ver/5.15.0 client-pltf-arch/x86_64 interactive/False from-script/False python/3.11.4 term/ (Linux 5.15.0-1074-azure),gzip(gfe)","destinationAttributes":{},"requestAttributes":{}},"resourceName":"projects/project-id","response":{"@type":"type.googleapis.com/google.iam.v1.Policy","bindings":[{"members":["serviceAccount:first@project-id.iam.gserviceaccount.com"],"role":"projects/project-id/roles/ThatRoleToo"},{"members":["serviceAccount:second@project-id.iam.gserviceaccount.com"],"role":"projects/project-id/roles/random"}],"etag":"BwYnQ8iRtu0="},"serviceData":{"@type":"type.googleapis.com/google.iam.v1.logging.AuditData","policyDelta":{"bindingDeltas":[{"action":"ADD","member":"serviceAccount:project-id@cloudservices.gserviceaccount.com","role":"roles/resourcemanager.projectIamAdmin"}]}},"serviceName":"cloudresourcemanager.googleapis.com","status":{}},"receiveTimestamp":"2024-11-19T13:12:21.785498724Z","resource":{"labels":{"project_id":"project-id"},"type":"project"},"severity":"NOTICE","timestamp":"2024-11-19T13:12:20.942393Z"} diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log-expected.json b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log-expected.json new file mode 100644 index 00000000000..0cb8e9d45a2 --- /dev/null +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log-expected.json @@ -0,0 +1,198 @@ +{ + "expected": [ + { + "@timestamp": "2024-11-19T13:12:20.942Z", + "client": { + "user": { + "email": "made-up-ci-account@project-id.iam.gserviceaccount.com", + "id": "serviceAccount:madeupprincipal@project-id.iam.gserviceaccount.com" + } + }, + "cloud": { + "project": { + "id": "project-id" + }, + "provider": "gcp" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "SetIamPolicy", + "id": "-30102re2sad8", + "kind": "event", + "original": "{\"insertId\":\"-30102re2sad8\",\"logName\":\"projects/project-id/logs/cloudaudit.googleapis.com%2Factivity\",\"protoPayload\":{\"@type\":\"type.googleapis.com/google.cloud.audit.AuditLog\",\"authenticationInfo\":{\"principalEmail\":\"made-up-ci-account@project-id.iam.gserviceaccount.com\",\"principalSubject\":\"serviceAccount:madeupprincipal@project-id.iam.gserviceaccount.com\",\"serviceAccountDelegationInfo\":[{\"principalSubject\":\"principal://iam.googleapis.com/projects/project-id/locations/global/workloadIdentityPools/...\"}]},\"authorizationInfo\":[{\"granted\":true,\"permission\":\"resourcemanager.projects.setIamPolicy\",\"permissionType\":\"ADMIN_WRITE\",\"resource\":\"projects/project-id\",\"resourceAttributes\":{\"name\":\"projects/project-id\",\"service\":\"cloudresourcemanager.googleapis.com\",\"type\":\"cloudresourcemanager.googleapis.com/Project\"}},{\"granted\":true,\"permission\":\"resourcemanager.projects.setIamPolicy\",\"permissionType\":\"ADMIN_WRITE\",\"resource\":\"projects/project-id\",\"resourceAttributes\":{\"name\":\"projects/project-id\",\"service\":\"cloudresourcemanager.googleapis.com\",\"type\":\"cloudresourcemanager.googleapis.com/Project\"}}],\"methodName\":\"SetIamPolicy\",\"request\":{\"@type\":\"type.googleapis.com/google.iam.v1.SetIamPolicyRequest\",\"policy\":{\"bindings\":[{\"members\":[\"serviceAccount:member-sa@project-id.iam.gserviceaccount.com\"],\"role\":\"projects/project-id/roles/ThatRoleToo\"},{\"members\":[\"serviceAccount:a@project-id.iam.gserviceaccount.com\"],\"role\":\"projects/project-id/roles/x\"},{\"members\":[\"serviceAccount:b@project-id.iam.gserviceaccount.com\"],\"role\":\"projects/project-id/roles/this_role_as_well\"},{\"members\":[\"serviceAccount:c@project-id.iam.gserviceaccount.com\",\"serviceAccount:d@project-id.iam.gserviceaccount.com\",\"serviceAccount:e@project-id.iam.gserviceaccount.com\"],\"role\":\"roles/browser\"},{\"members\":[\"serviceAccount:f@project-id.iam.gserviceaccount.com\",\"serviceAccount:g@project-id.iam.gserviceaccount.com\",\"serviceAccount:c@project-id.iam.gserviceaccount.com\"],\"role\":\"roles/cloudasset.viewer\"},{\"members\":[\"user:doesnotexist@elastic.co\"],\"role\":\"roles/cloudkms.admin\"},{\"members\":[\"group:agroup@elastic.co\"],\"role\":\"roles/owner\"}],\"etag\":\"BwYnObHBOBA=\"},\"resource\":\"project-id\"},\"requestMetadata\":{\"callerIp\":\"192.168.0.1\",\"callerSuppliedUserAgent\":\"google-cloud-sdk gcloud/501.0.0 command/gcloud.projects.add-iam-policy-binding invocation-id/e9e9e4b6f9294a7da9a2247dc101225a environment/None environment-version/None client-os/LINUX client-os-ver/5.15.0 client-pltf-arch/x86_64 interactive/False from-script/False python/3.11.4 term/ (Linux 5.15.0-1074-azure),gzip(gfe)\",\"destinationAttributes\":{},\"requestAttributes\":{}},\"resourceName\":\"projects/project-id\",\"response\":{\"@type\":\"type.googleapis.com/google.iam.v1.Policy\",\"bindings\":[{\"members\":[\"serviceAccount:first@project-id.iam.gserviceaccount.com\"],\"role\":\"projects/project-id/roles/ThatRoleToo\"},{\"members\":[\"serviceAccount:second@project-id.iam.gserviceaccount.com\"],\"role\":\"projects/project-id/roles/random\"}],\"etag\":\"BwYnQ8iRtu0=\"},\"serviceData\":{\"@type\":\"type.googleapis.com/google.iam.v1.logging.AuditData\",\"policyDelta\":{\"bindingDeltas\":[{\"action\":\"ADD\",\"member\":\"serviceAccount:project-id@cloudservices.gserviceaccount.com\",\"role\":\"roles/resourcemanager.projectIamAdmin\"}]}},\"serviceName\":\"cloudresourcemanager.googleapis.com\",\"status\":{}},\"receiveTimestamp\":\"2024-11-19T13:12:21.785498724Z\",\"resource\":{\"labels\":{\"project_id\":\"project-id\"},\"type\":\"project\"},\"severity\":\"NOTICE\",\"timestamp\":\"2024-11-19T13:12:20.942393Z\"}", + "outcome": "unknown", + "provider": "activity" + }, + "gcp": { + "audit": { + "authentication_info": { + "service_account_delegation_info": [ + { + "principalSubject": "principal://iam.googleapis.com/projects/project-id/locations/global/workloadIdentityPools/..." + } + ] + }, + "authorization_info": [ + { + "granted": true, + "permission": "resourcemanager.projects.setIamPolicy", + "permissionType": "ADMIN_WRITE", + "resource": "projects/project-id", + "resource_attributes": { + "name": "projects/project-id", + "service": "cloudresourcemanager.googleapis.com", + "type": "cloudresourcemanager.googleapis.com/Project" + } + }, + { + "granted": true, + "permission": "resourcemanager.projects.setIamPolicy", + "permissionType": "ADMIN_WRITE", + "resource": "projects/project-id", + "resource_attributes": { + "name": "projects/project-id", + "service": "cloudresourcemanager.googleapis.com", + "type": "cloudresourcemanager.googleapis.com/Project" + } + } + ], + "request": { + "@type": "type.googleapis.com/google.iam.v1.SetIamPolicyRequest", + "policy": { + "bindings": [ + { + "members": [ + "serviceAccount:member-sa@project-id.iam.gserviceaccount.com" + ], + "role": "projects/project-id/roles/ThatRoleToo" + }, + { + "members": [ + "serviceAccount:a@project-id.iam.gserviceaccount.com" + ], + "role": "projects/project-id/roles/x" + }, + { + "members": [ + "serviceAccount:b@project-id.iam.gserviceaccount.com" + ], + "role": "projects/project-id/roles/this_role_as_well" + }, + { + "members": [ + "serviceAccount:c@project-id.iam.gserviceaccount.com", + "serviceAccount:d@project-id.iam.gserviceaccount.com", + "serviceAccount:e@project-id.iam.gserviceaccount.com" + ], + "role": "roles/browser" + }, + { + "members": [ + "serviceAccount:f@project-id.iam.gserviceaccount.com", + "serviceAccount:g@project-id.iam.gserviceaccount.com", + "serviceAccount:c@project-id.iam.gserviceaccount.com" + ], + "role": "roles/cloudasset.viewer" + }, + { + "members": [ + "user:doesnotexist@elastic.co" + ], + "role": "roles/cloudkms.admin" + }, + { + "members": [ + "group:agroup@elastic.co" + ], + "role": "roles/owner" + } + ], + "etag": "BwYnObHBOBA=" + }, + "resource": "project-id" + }, + "resource_name": "projects/project-id", + "response": { + "@type": "type.googleapis.com/google.iam.v1.Policy", + "bindings": [ + { + "members": [ + "serviceAccount:first@project-id.iam.gserviceaccount.com" + ], + "role": "projects/project-id/roles/ThatRoleToo" + }, + { + "members": [ + "serviceAccount:second@project-id.iam.gserviceaccount.com" + ], + "role": "projects/project-id/roles/random" + } + ], + "etag": "BwYnQ8iRtu0=" + }, + "type": "type.googleapis.com/google.cloud.audit.AuditLog" + } + }, + "log": { + "level": "NOTICE", + "logger": "projects/project-id/logs/cloudaudit.googleapis.com%2Factivity" + }, + "related": { + "entity": [ + "projects/project-id/roles/x", + "roles/cloudasset.viewer", + "serviceAccount:member-sa@project-id.iam.gserviceaccount.com", + "serviceAccount:e@project-id.iam.gserviceaccount.com", + "serviceAccount:f@project-id.iam.gserviceaccount.com", + "serviceAccount:first@project-id.iam.gserviceaccount.com", + "projects/project-id/roles/random", + "serviceAccount:a@project-id.iam.gserviceaccount.com", + "serviceAccount:second@project-id.iam.gserviceaccount.com", + "projects/project-id", + "serviceAccount:g@project-id.iam.gserviceaccount.com", + "roles/cloudkms.admin", + "serviceAccount:b@project-id.iam.gserviceaccount.com", + "serviceAccount:d@project-id.iam.gserviceaccount.com", + "made-up-ci-account@project-id.iam.gserviceaccount.com", + "serviceAccount:c@project-id.iam.gserviceaccount.com", + "user:doesnotexist@elastic.co", + "projects/project-id/roles/ThatRoleToo", + "projects/project-id/roles/this_role_as_well", + "serviceAccount:madeupprincipal@project-id.iam.gserviceaccount.com", + "roles/owner", + "group:agroup@elastic.co", + "roles/browser" + ], + "ip": [ + "192.168.0.1" + ], + "user": [ + "made-up-ci-account@project-id.iam.gserviceaccount.com" + ] + }, + "service": { + "name": "cloudresourcemanager.googleapis.com" + }, + "source": { + "ip": "192.168.0.1" + }, + "tags": [ + "preserve_original_event" + ], + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Other", + "original": "google-cloud-sdk gcloud/501.0.0 command/gcloud.projects.add-iam-policy-binding invocation-id/e9e9e4b6f9294a7da9a2247dc101225a environment/None environment-version/None client-os/LINUX client-os-ver/5.15.0 client-pltf-arch/x86_64 interactive/False from-script/False python/3.11.4 term/ (Linux 5.15.0-1074-azure),gzip(gfe)", + "os": { + "full": "Linux 5.15.0", + "name": "Linux", + "version": "5.15.0" + } + } + } + ] +} diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log new file mode 100644 index 00000000000..65d6b535648 --- /dev/null +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log @@ -0,0 +1 @@ +{"insertId":"-w5vrlhdm7gk","labels":{"compute.googleapis.com/root_trigger_id":"UUID"},"logName":"projects/project-id/logs/cloudaudit.googleapis.com%2Factivity","operation":{"first":true,"id":"operation-1732021993132-62743cba4d27a-d29b55ba-2cd69d6a","producer":"compute.googleapis.com"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"project-id@cloudservices.gserviceaccount.com","principalSubject":"serviceAccount:project-id@cloudservices.gserviceaccount.com","serviceAccountDelegationInfo":[{"firstPartyPrincipal":{"principalEmail":"principalA@prod.google.com"}}]},"authorizationInfo":[{"granted":true,"permission":"compute.instances.create","permissionType":"ADMIN_WRITE","resource":"projects/project-id/zones/us-central1-a/instances/x-logs","resourceAttributes":{"name":"projects/project-id/zones/us-central1-a/instances/x-logs","service":"compute","type":"compute.instances"}},{"granted":true,"permission":"compute.disks.create","permissionType":"ADMIN_WRITE","resource":"projects/project-id/zones/us-central1-a/disks/x-logs","resourceAttributes":{"name":"projects/project-id/zones/us-central1-a/disks/x-logs","service":"compute","type":"compute.disks"}},{"granted":true,"permission":"compute.subnetworks.use","permissionType":"ADMIN_WRITE","resource":"projects/project-id/regions/us-central1/subnetworks/x-logs-network","resourceAttributes":{"name":"projects/project-id/regions/us-central1/subnetworks/x-logs-network","service":"compute","type":"compute.subnetworks"}},{"granted":true,"permission":"compute.subnetworks.useExternalIp","permissionType":"ADMIN_WRITE","resource":"projects/project-id/regions/us-central1/subnetworks/x-logs-network","resourceAttributes":{"name":"projects/project-id/regions/us-central1/subnetworks/x-logs-network","service":"compute","type":"compute.subnetworks"}},{"granted":true,"permission":"compute.instances.setMetadata","permissionType":"ADMIN_WRITE","resource":"projects/project-id/zones/us-central1-a/instances/x-logs","resourceAttributes":{"name":"projects/project-id/zones/us-central1-a/instances/x-logs","service":"compute","type":"compute.instances"}},{"granted":true,"permission":"compute.instances.setLabels","permissionType":"ADMIN_WRITE","resource":"projects/project-id/zones/us-central1-a/instances/x-logs","resourceAttributes":{"name":"projects/project-id/zones/us-central1-a/instances/x-logs","service":"compute","type":"compute.instances"}},{"granted":true,"permission":"compute.instances.setServiceAccount","permissionType":"ADMIN_WRITE","resource":"projects/project-id/zones/us-central1-a/instances/x-logs","resourceAttributes":{"name":"projects/project-id/zones/us-central1-a/instances/x-logs","service":"compute","type":"compute.instances"}}],"metadata":{"usedResources":{"attachedDisks":[{"isBootDisk":true,"sourceImage":"https://www.googleapis.com/compute/v1/projects/global-project/global/images/ubuntu-minimal-2204-jammy-v20241115","sourceImageId":"source-image-id"}]}},"methodName":"v1.compute.instances.insert","request":{"@type":"type.googleapis.com/compute.instances.insert","disks":[{"autoDelete":true,"boot":true,"deviceName":"boot","initializeParams":{"sourceImage":"https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/family/ubuntu-minimal-2204-lts"},"type":"PERSISTENT"}],"machineType":"https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/machineTypes/n2-standard-4","name":"x-logs","networkInterfaces":[{"accessConfigs":[{"name":"External NAT","type":"ONE_TO_ONE_NAT"}],"network":"https://www.googleapis.com/compute/v1/projects/project-id/global/networks/x-logs-network"}],"serviceAccounts":[{"email":"x-logs-sa@project-id.iam.gserviceaccount.com","scopes":["https://www.googleapis.com/auth/cloud-platform","https://www.googleapis.com/auth/cloudplatformorganizations"]}]},"requestMetadata":{"callerIp":"175.16.199.45","callerSuppliedUserAgent":"Google-Deployment-Manager,gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2024-11-19T13:13:13.966817Z"}},"resourceLocation":{"currentLocations":["us-central1-a"]},"resourceName":"projects/project-id/zones/us-central1-a/instances/x-logs","response":{"@type":"type.googleapis.com/operation","id":"id","insertTime":"2024-11-19T05:13:13.857-08:00","name":"operation-id","operationType":"insert","progress":"0","selfLink":"https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/operations/operation-id","selfLinkWithId":"https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/operations/","startTime":"2024-11-19T05:13:13.857-08:00","status":"RUNNING","targetId":"targetId","targetLink":"https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/instances/x-logs","user":"project-id@cloudservices.gserviceaccount.com","zone":"https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a"},"serviceName":"compute.googleapis.com"},"receiveTimestamp":"2024-11-19T13:13:14.634438657Z","resource":{"labels":{"instance_id":"2525602744967966726","project_id":"project-id","zone":"us-central1-a"},"type":"gce_instance"},"severity":"NOTICE","timestamp":"2024-11-19T13:13:13.176899Z"} diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log-expected.json b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log-expected.json new file mode 100644 index 00000000000..380b316f162 --- /dev/null +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log-expected.json @@ -0,0 +1,254 @@ +{ + "expected": [ + { + "@timestamp": "2024-11-19T13:13:13.176Z", + "client": { + "user": { + "email": "project-id@cloudservices.gserviceaccount.com", + "id": "serviceAccount:project-id@cloudservices.gserviceaccount.com" + } + }, + "cloud": { + "instance": { + "id": "2525602744967966726" + }, + "project": { + "id": "project-id" + }, + "provider": "gcp" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "v1.compute.instances.insert", + "category": [ + "session" + ], + "id": "-w5vrlhdm7gk", + "kind": "event", + "original": "{\"insertId\":\"-w5vrlhdm7gk\",\"labels\":{\"compute.googleapis.com/root_trigger_id\":\"UUID\"},\"logName\":\"projects/project-id/logs/cloudaudit.googleapis.com%2Factivity\",\"operation\":{\"first\":true,\"id\":\"operation-1732021993132-62743cba4d27a-d29b55ba-2cd69d6a\",\"producer\":\"compute.googleapis.com\"},\"protoPayload\":{\"@type\":\"type.googleapis.com/google.cloud.audit.AuditLog\",\"authenticationInfo\":{\"principalEmail\":\"project-id@cloudservices.gserviceaccount.com\",\"principalSubject\":\"serviceAccount:project-id@cloudservices.gserviceaccount.com\",\"serviceAccountDelegationInfo\":[{\"firstPartyPrincipal\":{\"principalEmail\":\"principalA@prod.google.com\"}}]},\"authorizationInfo\":[{\"granted\":true,\"permission\":\"compute.instances.create\",\"permissionType\":\"ADMIN_WRITE\",\"resource\":\"projects/project-id/zones/us-central1-a/instances/x-logs\",\"resourceAttributes\":{\"name\":\"projects/project-id/zones/us-central1-a/instances/x-logs\",\"service\":\"compute\",\"type\":\"compute.instances\"}},{\"granted\":true,\"permission\":\"compute.disks.create\",\"permissionType\":\"ADMIN_WRITE\",\"resource\":\"projects/project-id/zones/us-central1-a/disks/x-logs\",\"resourceAttributes\":{\"name\":\"projects/project-id/zones/us-central1-a/disks/x-logs\",\"service\":\"compute\",\"type\":\"compute.disks\"}},{\"granted\":true,\"permission\":\"compute.subnetworks.use\",\"permissionType\":\"ADMIN_WRITE\",\"resource\":\"projects/project-id/regions/us-central1/subnetworks/x-logs-network\",\"resourceAttributes\":{\"name\":\"projects/project-id/regions/us-central1/subnetworks/x-logs-network\",\"service\":\"compute\",\"type\":\"compute.subnetworks\"}},{\"granted\":true,\"permission\":\"compute.subnetworks.useExternalIp\",\"permissionType\":\"ADMIN_WRITE\",\"resource\":\"projects/project-id/regions/us-central1/subnetworks/x-logs-network\",\"resourceAttributes\":{\"name\":\"projects/project-id/regions/us-central1/subnetworks/x-logs-network\",\"service\":\"compute\",\"type\":\"compute.subnetworks\"}},{\"granted\":true,\"permission\":\"compute.instances.setMetadata\",\"permissionType\":\"ADMIN_WRITE\",\"resource\":\"projects/project-id/zones/us-central1-a/instances/x-logs\",\"resourceAttributes\":{\"name\":\"projects/project-id/zones/us-central1-a/instances/x-logs\",\"service\":\"compute\",\"type\":\"compute.instances\"}},{\"granted\":true,\"permission\":\"compute.instances.setLabels\",\"permissionType\":\"ADMIN_WRITE\",\"resource\":\"projects/project-id/zones/us-central1-a/instances/x-logs\",\"resourceAttributes\":{\"name\":\"projects/project-id/zones/us-central1-a/instances/x-logs\",\"service\":\"compute\",\"type\":\"compute.instances\"}},{\"granted\":true,\"permission\":\"compute.instances.setServiceAccount\",\"permissionType\":\"ADMIN_WRITE\",\"resource\":\"projects/project-id/zones/us-central1-a/instances/x-logs\",\"resourceAttributes\":{\"name\":\"projects/project-id/zones/us-central1-a/instances/x-logs\",\"service\":\"compute\",\"type\":\"compute.instances\"}}],\"metadata\":{\"usedResources\":{\"attachedDisks\":[{\"isBootDisk\":true,\"sourceImage\":\"https://www.googleapis.com/compute/v1/projects/global-project/global/images/ubuntu-minimal-2204-jammy-v20241115\",\"sourceImageId\":\"source-image-id\"}]}},\"methodName\":\"v1.compute.instances.insert\",\"request\":{\"@type\":\"type.googleapis.com/compute.instances.insert\",\"disks\":[{\"autoDelete\":true,\"boot\":true,\"deviceName\":\"boot\",\"initializeParams\":{\"sourceImage\":\"https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/family/ubuntu-minimal-2204-lts\"},\"type\":\"PERSISTENT\"}],\"machineType\":\"https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/machineTypes/n2-standard-4\",\"name\":\"x-logs\",\"networkInterfaces\":[{\"accessConfigs\":[{\"name\":\"External NAT\",\"type\":\"ONE_TO_ONE_NAT\"}],\"network\":\"https://www.googleapis.com/compute/v1/projects/project-id/global/networks/x-logs-network\"}],\"serviceAccounts\":[{\"email\":\"x-logs-sa@project-id.iam.gserviceaccount.com\",\"scopes\":[\"https://www.googleapis.com/auth/cloud-platform\",\"https://www.googleapis.com/auth/cloudplatformorganizations\"]}]},\"requestMetadata\":{\"callerIp\":\"175.16.199.45\",\"callerSuppliedUserAgent\":\"Google-Deployment-Manager,gzip(gfe)\",\"destinationAttributes\":{},\"requestAttributes\":{\"auth\":{},\"time\":\"2024-11-19T13:13:13.966817Z\"}},\"resourceLocation\":{\"currentLocations\":[\"us-central1-a\"]},\"resourceName\":\"projects/project-id/zones/us-central1-a/instances/x-logs\",\"response\":{\"@type\":\"type.googleapis.com/operation\",\"id\":\"id\",\"insertTime\":\"2024-11-19T05:13:13.857-08:00\",\"name\":\"operation-id\",\"operationType\":\"insert\",\"progress\":\"0\",\"selfLink\":\"https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/operations/operation-id\",\"selfLinkWithId\":\"https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/operations/\",\"startTime\":\"2024-11-19T05:13:13.857-08:00\",\"status\":\"RUNNING\",\"targetId\":\"targetId\",\"targetLink\":\"https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/instances/x-logs\",\"user\":\"project-id@cloudservices.gserviceaccount.com\",\"zone\":\"https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a\"},\"serviceName\":\"compute.googleapis.com\"},\"receiveTimestamp\":\"2024-11-19T13:13:14.634438657Z\",\"resource\":{\"labels\":{\"instance_id\":\"2525602744967966726\",\"project_id\":\"project-id\",\"zone\":\"us-central1-a\"},\"type\":\"gce_instance\"},\"severity\":\"NOTICE\",\"timestamp\":\"2024-11-19T13:13:13.176899Z\"}", + "outcome": "unknown", + "provider": "activity", + "type": [ + "start" + ] + }, + "gcp": { + "audit": { + "authentication_info": { + "service_account_delegation_info": [ + { + "firstPartyPrincipal": { + "principalEmail": "principalA@prod.google.com" + } + } + ] + }, + "authorization_info": [ + { + "granted": true, + "permission": "compute.instances.create", + "permissionType": "ADMIN_WRITE", + "resource": "projects/project-id/zones/us-central1-a/instances/x-logs", + "resource_attributes": { + "name": "projects/project-id/zones/us-central1-a/instances/x-logs", + "service": "compute", + "type": "compute.instances" + } + }, + { + "granted": true, + "permission": "compute.disks.create", + "permissionType": "ADMIN_WRITE", + "resource": "projects/project-id/zones/us-central1-a/disks/x-logs", + "resource_attributes": { + "name": "projects/project-id/zones/us-central1-a/disks/x-logs", + "service": "compute", + "type": "compute.disks" + } + }, + { + "granted": true, + "permission": "compute.subnetworks.use", + "permissionType": "ADMIN_WRITE", + "resource": "projects/project-id/regions/us-central1/subnetworks/x-logs-network", + "resource_attributes": { + "name": "projects/project-id/regions/us-central1/subnetworks/x-logs-network", + "service": "compute", + "type": "compute.subnetworks" + } + }, + { + "granted": true, + "permission": "compute.subnetworks.useExternalIp", + "permissionType": "ADMIN_WRITE", + "resource": "projects/project-id/regions/us-central1/subnetworks/x-logs-network", + "resource_attributes": { + "name": "projects/project-id/regions/us-central1/subnetworks/x-logs-network", + "service": "compute", + "type": "compute.subnetworks" + } + }, + { + "granted": true, + "permission": "compute.instances.setMetadata", + "permissionType": "ADMIN_WRITE", + "resource": "projects/project-id/zones/us-central1-a/instances/x-logs", + "resource_attributes": { + "name": "projects/project-id/zones/us-central1-a/instances/x-logs", + "service": "compute", + "type": "compute.instances" + } + }, + { + "granted": true, + "permission": "compute.instances.setLabels", + "permissionType": "ADMIN_WRITE", + "resource": "projects/project-id/zones/us-central1-a/instances/x-logs", + "resource_attributes": { + "name": "projects/project-id/zones/us-central1-a/instances/x-logs", + "service": "compute", + "type": "compute.instances" + } + }, + { + "granted": true, + "permission": "compute.instances.setServiceAccount", + "permissionType": "ADMIN_WRITE", + "resource": "projects/project-id/zones/us-central1-a/instances/x-logs", + "resource_attributes": { + "name": "projects/project-id/zones/us-central1-a/instances/x-logs", + "service": "compute", + "type": "compute.instances" + } + } + ], + "labels": { + "compute.googleapis.com/root_trigger_id": "UUID" + }, + "logentry_operation": { + "id": "operation-1732021993132-62743cba4d27a-d29b55ba-2cd69d6a" + }, + "metadata": { + "usedResources": { + "attachedDisks": [ + { + "isBootDisk": true, + "sourceImage": "https://www.googleapis.com/compute/v1/projects/global-project/global/images/ubuntu-minimal-2204-jammy-v20241115", + "sourceImageId": "source-image-id" + } + ] + } + }, + "request": { + "@type": "type.googleapis.com/compute.instances.insert", + "disks": [ + { + "autoDelete": true, + "boot": true, + "deviceName": "boot", + "initializeParams": { + "sourceImage": "https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/family/ubuntu-minimal-2204-lts" + }, + "type": "PERSISTENT" + } + ], + "machineType": "https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/machineTypes/n2-standard-4", + "name": "x-logs", + "networkInterfaces": [ + { + "accessConfigs": [ + { + "name": "External NAT", + "type": "ONE_TO_ONE_NAT" + } + ], + "network": "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/x-logs-network" + } + ], + "serviceAccounts": [ + { + "email": "x-logs-sa@project-id.iam.gserviceaccount.com", + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloudplatformorganizations" + ] + } + ] + }, + "resource_location": { + "current_locations": [ + "us-central1-a" + ] + }, + "resource_name": "projects/project-id/zones/us-central1-a/instances/x-logs", + "response": { + "@type": "type.googleapis.com/operation", + "id": "id", + "insertTime": "2024-11-19T05:13:13.857-08:00", + "name": "operation-id", + "operationType": "insert", + "progress": "0", + "selfLink": "https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/operations/operation-id", + "selfLinkWithId": "https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/operations/", + "startTime": "2024-11-19T05:13:13.857-08:00", + "status_value": "RUNNING", + "targetId": "targetId", + "targetLink": "https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/instances/x-logs", + "user": "project-id@cloudservices.gserviceaccount.com", + "zone": "https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a" + }, + "type": "type.googleapis.com/google.cloud.audit.AuditLog" + } + }, + "log": { + "level": "NOTICE", + "logger": "projects/project-id/logs/cloudaudit.googleapis.com%2Factivity" + }, + "related": { + "entity": [ + "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/x-logs-network", + "projects/project-id/zones/us-central1-a/instances/x-logs", + "serviceAccount:project-id@cloudservices.gserviceaccount.com", + "project-id@cloudservices.gserviceaccount.com", + "x-logs-sa@project-id.iam.gserviceaccount.com" + ], + "ip": [ + "175.16.199.45" + ], + "user": [ + "project-id@cloudservices.gserviceaccount.com" + ] + }, + "service": { + "name": "compute.googleapis.com" + }, + "source": { + "geo": { + "city_name": "Changchun", + "continent_name": "Asia", + "country_iso_code": "CN", + "country_name": "China", + "location": { + "lat": 43.88, + "lon": 125.3228 + }, + "region_iso_code": "CN-22", + "region_name": "Jilin Sheng" + }, + "ip": "175.16.199.45" + }, + "tags": [ + "preserve_original_event" + ], + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Other", + "original": "Google-Deployment-Manager,gzip(gfe)" + } + } + ] +} diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-iamcredentials-googleapis-com.log b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-iamcredentials-googleapis-com.log new file mode 100644 index 00000000000..976a77cef60 --- /dev/null +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-iamcredentials-googleapis-com.log @@ -0,0 +1 @@ +{"insertId":"15djrryd6bap","logName":"projects/project-id/logs/cloudaudit.googleapis.com%2Fdata_access","operation":{"first":true,"id":"","last":true,"producer":"iamcredentials.googleapis.com"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalSubject":"principal://iam.googleapis.com/projects/project-id/locations/global/workloadIdentityPools/...","serviceAccountDelegationInfo":[{}]},"authorizationInfo":[{"granted":true,"permission":"iam.serviceAccounts.getAccessToken","permissionType":"ADMIN_READ","resourceAttributes":{}}],"metadata":{"identityDelegationChain":["projects/-/serviceAccounts/made-up-ci-account@project-id.iam.gserviceaccount.com"]},"methodName":"GenerateAccessToken","request":{"@type":"type.googleapis.com/google.iam.credentials.v1.GenerateAccessTokenRequest","name":"projects/-/serviceAccounts/made-up-ci-account@project-id.iam.gserviceaccount.com"},"requestMetadata":{"callerIp":"175.16.199.45","callerSuppliedUserAgent":"Go-http-client/2.0,gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2024-11-19T00:49:55.301834867Z"}},"resourceName":"projects/-/serviceAccounts/somenumber","serviceName":"iamcredentials.googleapis.com","status":{}},"receiveTimestamp":"2024-11-19T00:49:56.551702143Z","resource":{"labels":{"email_id":"made-up-ci-account@project-id.iam.gserviceaccount.com","project_id":"project-id","unique_id":"somenumber"},"type":"service_account"},"severity":"INFO","timestamp":"2024-11-19T00:49:55.293368631Z"} diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-iamcredentials-googleapis-com.log-expected.json b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-iamcredentials-googleapis-com.log-expected.json new file mode 100644 index 00000000000..3c1d622e611 --- /dev/null +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-iamcredentials-googleapis-com.log-expected.json @@ -0,0 +1,105 @@ +{ + "expected": [ + { + "@timestamp": "2024-11-19T00:49:55.293Z", + "client": { + "user": { + "id": "principal://iam.googleapis.com/projects/project-id/locations/global/workloadIdentityPools/..." + } + }, + "cloud": { + "project": { + "id": "project-id" + }, + "provider": "gcp" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "GenerateAccessToken", + "category": [ + "network", + "configuration" + ], + "id": "15djrryd6bap", + "kind": "event", + "original": "{\"insertId\":\"15djrryd6bap\",\"logName\":\"projects/project-id/logs/cloudaudit.googleapis.com%2Fdata_access\",\"operation\":{\"first\":true,\"id\":\"\",\"last\":true,\"producer\":\"iamcredentials.googleapis.com\"},\"protoPayload\":{\"@type\":\"type.googleapis.com/google.cloud.audit.AuditLog\",\"authenticationInfo\":{\"principalSubject\":\"principal://iam.googleapis.com/projects/project-id/locations/global/workloadIdentityPools/...\",\"serviceAccountDelegationInfo\":[{}]},\"authorizationInfo\":[{\"granted\":true,\"permission\":\"iam.serviceAccounts.getAccessToken\",\"permissionType\":\"ADMIN_READ\",\"resourceAttributes\":{}}],\"metadata\":{\"identityDelegationChain\":[\"projects/-/serviceAccounts/made-up-ci-account@project-id.iam.gserviceaccount.com\"]},\"methodName\":\"GenerateAccessToken\",\"request\":{\"@type\":\"type.googleapis.com/google.iam.credentials.v1.GenerateAccessTokenRequest\",\"name\":\"projects/-/serviceAccounts/made-up-ci-account@project-id.iam.gserviceaccount.com\"},\"requestMetadata\":{\"callerIp\":\"175.16.199.45\",\"callerSuppliedUserAgent\":\"Go-http-client/2.0,gzip(gfe)\",\"destinationAttributes\":{},\"requestAttributes\":{\"auth\":{},\"time\":\"2024-11-19T00:49:55.301834867Z\"}},\"resourceName\":\"projects/-/serviceAccounts/somenumber\",\"serviceName\":\"iamcredentials.googleapis.com\",\"status\":{}},\"receiveTimestamp\":\"2024-11-19T00:49:56.551702143Z\",\"resource\":{\"labels\":{\"email_id\":\"made-up-ci-account@project-id.iam.gserviceaccount.com\",\"project_id\":\"project-id\",\"unique_id\":\"somenumber\"},\"type\":\"service_account\"},\"severity\":\"INFO\",\"timestamp\":\"2024-11-19T00:49:55.293368631Z\"}", + "outcome": "success", + "provider": "data_access", + "type": [ + "access", + "allowed" + ] + }, + "gcp": { + "audit": { + "authorization_info": [ + { + "granted": true, + "permission": "iam.serviceAccounts.getAccessToken", + "permissionType": "ADMIN_READ" + } + ], + "logentry_operation": { + "id": "" + }, + "metadata": { + "identityDelegationChain": [ + "projects/-/serviceAccounts/made-up-ci-account@project-id.iam.gserviceaccount.com" + ] + }, + "request": { + "@type": "type.googleapis.com/google.iam.credentials.v1.GenerateAccessTokenRequest", + "name": "projects/-/serviceAccounts/made-up-ci-account@project-id.iam.gserviceaccount.com" + }, + "resource_name": "projects/-/serviceAccounts/somenumber", + "type": "type.googleapis.com/google.cloud.audit.AuditLog" + } + }, + "log": { + "level": "INFO", + "logger": "projects/project-id/logs/cloudaudit.googleapis.com%2Fdata_access" + }, + "related": { + "entity": [ + "projects/-/serviceAccounts/made-up-ci-account@project-id.iam.gserviceaccount.com", + "principal://iam.googleapis.com/projects/project-id/locations/global/workloadIdentityPools/...", + "projects/-/serviceAccounts/somenumber" + ], + "ip": [ + "175.16.199.45" + ] + }, + "service": { + "name": "iamcredentials.googleapis.com" + }, + "source": { + "geo": { + "city_name": "Changchun", + "continent_name": "Asia", + "country_iso_code": "CN", + "country_name": "China", + "location": { + "lat": 43.88, + "lon": 125.3228 + }, + "region_iso_code": "CN-22", + "region_name": "Jilin Sheng" + }, + "ip": "175.16.199.45" + }, + "tags": [ + "preserve_original_event" + ], + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Go-http-client", + "original": "Go-http-client/2.0,gzip(gfe)", + "version": "2.0" + } + } + ] +} diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-sdh-3695.log-expected.json b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-sdh-3695.log-expected.json index fb79f53d17a..1e08be1feda 100644 --- a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-sdh-3695.log-expected.json +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-sdh-3695.log-expected.json @@ -40,6 +40,10 @@ "logger": "organizations/123456789098/logs/cloudaudit.googleapis.com%2Fdata_access" }, "related": { + "entity": [ + "organizations/123456789098", + "joel.miller@contoso.com" + ], "user": [ "joel.miller@contoso.com" ] @@ -52,4 +56,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index e705b8f329a..5d1f8480fdc 100644 --- a/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -48,7 +48,7 @@ processors: pattern: "%{}%2F%{event.provider}" ignore_missing: true # NOTE test data fails the spec - ignore_failure: true + ignore_failure: true - set: field: event.kind @@ -105,6 +105,118 @@ processors: API_VERSION: (v\d+([a-z]+)?(\d+)?) RESOURCE_TYPE: ([a-z]+((\.[a-z0-9]+)+)?) ignore_missing: true + - script: + description: Appends any relevant entity to `related.entity` for all events + lang: painless + on_failure: + - set: + description: Adds error reason to the document + field: error.message + value: "{{{ _ingest.on_failure_message }}}" + source: | + void addFields(Set entities, String[] fields) { + for (String field : fields) { + addField(entities, field); + } + } + + void addField(Set entities, String fieldName) { + addValue(entities, field(fieldName).get(null)); + } + + boolean addValues(Set entities, String[] values) { + boolean addedAll = true; + for (String value : values) { + addedAll = addedAll && addValue(entities, value); + } + + return addedAll; + } + + boolean addValue(Set entities, String value) { + if (value == null || value == "") { + return false; + } + + return entities.add(value); + } + + boolean isKubernetes = false; + if (ctx.json?.resource?.type != null) { + String resType = ctx.json.resource.type; + if (resType == "k8s_cluster" || resType == "gke_cluster" || resType == "kubernetes") { + isKubernetes = true; + } + } + + // Using tree set to ensure a sorting is kept (testing purposes) + TreeSet entities = new TreeSet(); + + addField(entities, "json.proyoPayload.request.parent"); + if (!isKubernetes) { + addFields(entities, new String[]{ + "json.protoPayload.resourceName", + "json.protoPayload.response.user" + }); + } + + + HashMap authInfo = field("json.protoPayload.authenticationInfo").get(new HashMap()); + if (!isKubernetes) { + addValue(entities, authInfo.principalEmail); + } + addValues(entities, new String[]{ + authInfo.principalSubject, + authInfo.serviceAccountKeyName + }); + if (authInfo.serviceAccountDelegationInfo != null) { + if (!(authInfo.serviceAccountDelegationInfo instanceof List)) { + authInfo.serviceAccountDelegationInfo.get(new ArrayList()).stream().forEach( + delegInfo -> { + addValues(entities, new String[]{ + delegInfo.principalSubject, + delegInfo.firstPartyPrincipal.principalEmail, + delegInfo.thirdPartyPrincipal.principalEmail + }); + } + ); + } + } + + String serviceName = field("json.protoPayload.serviceName").get(null); + + if (serviceName == "compute.googleapis.com") { + field("json.protoPayload.request.networkInterfaces").get(new ArrayList()).stream().forEach( + f -> addValue(entities, f.network) + ); + + field("json.protoPayload.request.serviceAccounts").get(new ArrayList()).stream().forEach( + f -> addValue(entities, f.email) + ); + + field("json.protoPayload.request.disks").get(new ArrayList()).stream().forEach( + f -> addValue(entities, f.source) + ); + } else if (serviceName == "cloudresourcemanager.googleapis.com") { + field("json.protoPayload.request.policy.bindings").get(new ArrayList()).stream().forEach( + f -> { + addValue(entities, f.role); + f.members.forEach(m -> addValue(entities, m)); + } + ); + field("json.protoPayload.response.bindings").get(new ArrayList()).stream().forEach( + f -> { + addValue(entities, f.role); + f.members.forEach(m -> addValue(entities, m)); + } + ); + } else if (serviceName == "iamcredentials.googleapis.com") { + field("json.protoPayload.metadata.identityDelegationChain").get(new ArrayList()).stream().forEach( + f -> addValue(entities, f) + ); + } + + field("related.entity").set(entities); ## # AuthenticationInfo @@ -370,9 +482,9 @@ processors: ## # if gcp.audit.authorization_info.[0].granted is true then -# set event.category [network, configuration] and event.type to [access, allowed]; +# set event.category [network, configuration] and event.type to [access, allowed]; # Caveat -# 1. protoPayload.resourceName is a single value while authorization_info[].resource +# 1. protoPayload.resourceName is a single value while authorization_info[].resource # is a list. # 2. as per test data authorization_info may not be as per spec. ## diff --git a/packages/gcp/data_stream/audit/fields/base-fields.yml b/packages/gcp/data_stream/audit/fields/base-fields.yml index 4a7da765108..529f2bfaeee 100644 --- a/packages/gcp/data_stream/audit/fields/base-fields.yml +++ b/packages/gcp/data_stream/audit/fields/base-fields.yml @@ -18,3 +18,10 @@ type: constant_keyword description: Event dataset value: gcp.audit +- name: related.entity + description: | + A collection of all entity identifiers associated with the document. + If the document contains multiple entities, identifiers for each will be included. + Example identifiers include (but not limited to) cloud resource IDs, email addresses, + and hostnames. + type: keyword diff --git a/packages/gcp/data_stream/audit/fields/fields.yml b/packages/gcp/data_stream/audit/fields/fields.yml index 19a30dc6a55..ee4885c98eb 100644 --- a/packages/gcp/data_stream/audit/fields/fields.yml +++ b/packages/gcp/data_stream/audit/fields/fields.yml @@ -113,13 +113,6 @@ - name: policyType type: keyword description: "Indicates the type of the policy." - - name: related.entity - description: | - A collection of all entity identifiers associated with the document. - If the document contains multiple entities, identifiers for each will be included. - Example identifiers include (but not limited to) cloud resource IDs, email addresses, - and hostnames. - type: keyword - name: request type: flattened - name: request_metadata diff --git a/packages/gcp/docs/README.md b/packages/gcp/docs/README.md index 2f542771070..040dea358f3 100644 --- a/packages/gcp/docs/README.md +++ b/packages/gcp/docs/README.md @@ -288,6 +288,7 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | +| related.entity | A collection of all entity identifiers associated with the document. If the document contains multiple entities, identifiers for each will be included. Example identifiers include (but not limited to) cloud resource IDs, email addresses, and hostnames. | keyword | An example event for `audit` looks as following: diff --git a/packages/gcp/docs/audit.md b/packages/gcp/docs/audit.md index 44faed19057..c2d8be56e80 100644 --- a/packages/gcp/docs/audit.md +++ b/packages/gcp/docs/audit.md @@ -79,6 +79,7 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | log.offset | Log offset | long | | related.entity | A collection of all entity identifiers associated with the document. If the document contains multiple entities, identifiers for each will be included. Example identifiers include (but not limited to) cloud resource IDs, email addresses, and hostnames. | keyword | + An example event for `audit` looks as following: ```json @@ -203,4 +204,4 @@ An example event for `audit` looks as following: "version": "71.0." } } -``` +``` \ No newline at end of file diff --git a/packages/gcp/manifest.yml b/packages/gcp/manifest.yml index 64276f363b5..539c0e69fdb 100644 --- a/packages/gcp/manifest.yml +++ b/packages/gcp/manifest.yml @@ -1,6 +1,6 @@ name: gcp title: Google Cloud Platform -version: "2.38.0" +version: "2.39.0" description: Collect logs and metrics from Google Cloud Platform with Elastic Agent. type: integration icons: From 71de5e8f1e2a4dcd3a626d2f5a52b98c569b7959 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Tue, 19 Nov 2024 17:06:00 +0100 Subject: [PATCH 3/6] update changelog entry --- packages/gcp/changelog.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/gcp/changelog.yml b/packages/gcp/changelog.yml index 696bd94890f..8987791bf8a 100644 --- a/packages/gcp/changelog.yml +++ b/packages/gcp/changelog.yml @@ -1,9 +1,9 @@ # newer versions go on top - version: "2.39.0" changes: - - description: WIP KUBA DEBUG - Adding related.entity + - description: Add `related.entity` field to audit logs. type: enhancement - link: TBD + link: https://github.com/elastic/integrations/pull/11762 - version: "2.38.0" changes: - description: Add `policy_violation_info`, `metadata` and `related` fields to audit logs. From ee85a15cdd4ec875dafc001e7e6eb11c94bc623f Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 22 Nov 2024 12:44:39 +0100 Subject: [PATCH 4/6] fix code smells --- .../elasticsearch/ingest_pipeline/default.yml | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 5d1f8480fdc..6734e8db82e 100644 --- a/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -138,7 +138,7 @@ processors: return false; } - return entities.add(value); + return entities.add(value); } boolean isKubernetes = false; @@ -152,7 +152,7 @@ processors: // Using tree set to ensure a sorting is kept (testing purposes) TreeSet entities = new TreeSet(); - addField(entities, "json.proyoPayload.request.parent"); + addField(entities, "json.protoPayload.request.parent"); if (!isKubernetes) { addFields(entities, new String[]{ "json.protoPayload.resourceName", @@ -169,18 +169,16 @@ processors: authInfo.principalSubject, authInfo.serviceAccountKeyName }); - if (authInfo.serviceAccountDelegationInfo != null) { - if (!(authInfo.serviceAccountDelegationInfo instanceof List)) { - authInfo.serviceAccountDelegationInfo.get(new ArrayList()).stream().forEach( - delegInfo -> { - addValues(entities, new String[]{ - delegInfo.principalSubject, - delegInfo.firstPartyPrincipal.principalEmail, - delegInfo.thirdPartyPrincipal.principalEmail - }); - } - ); - } + if (authInfo.serviceAccountDelegationInfo != null && !(authInfo.serviceAccountDelegationInfo instanceof List)) { + authInfo.serviceAccountDelegationInfo.get(new ArrayList()).stream().forEach( + delegInfo -> { + addValues(entities, new String[]{ + delegInfo.principalSubject, + delegInfo.firstPartyPrincipal?.principalEmail, + delegInfo.thirdPartyPrincipal?.principalEmail + }); + } + ); } String serviceName = field("json.protoPayload.serviceName").get(null); From dad99d4b6bdbc6ea1a07256769a75c5704f8a32b Mon Sep 17 00:00:00 2001 From: kubasobon Date: Fri, 22 Nov 2024 12:47:00 +0100 Subject: [PATCH 5/6] use List instead of []String --- .../elasticsearch/ingest_pipeline/default.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 6734e8db82e..9c986140faa 100644 --- a/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -114,7 +114,7 @@ processors: field: error.message value: "{{{ _ingest.on_failure_message }}}" source: | - void addFields(Set entities, String[] fields) { + void addFields(Set entities, List fields) { for (String field : fields) { addField(entities, field); } @@ -124,7 +124,7 @@ processors: addValue(entities, field(fieldName).get(null)); } - boolean addValues(Set entities, String[] values) { + boolean addValues(Set entities, List values) { boolean addedAll = true; for (String value : values) { addedAll = addedAll && addValue(entities, value); @@ -154,10 +154,10 @@ processors: addField(entities, "json.protoPayload.request.parent"); if (!isKubernetes) { - addFields(entities, new String[]{ + addFields(entities, [ "json.protoPayload.resourceName", "json.protoPayload.response.user" - }); + ]); } @@ -165,18 +165,18 @@ processors: if (!isKubernetes) { addValue(entities, authInfo.principalEmail); } - addValues(entities, new String[]{ + addValues(entities, [ authInfo.principalSubject, authInfo.serviceAccountKeyName - }); + ]); if (authInfo.serviceAccountDelegationInfo != null && !(authInfo.serviceAccountDelegationInfo instanceof List)) { authInfo.serviceAccountDelegationInfo.get(new ArrayList()).stream().forEach( delegInfo -> { - addValues(entities, new String[]{ + addValues(entities, [ delegInfo.principalSubject, delegInfo.firstPartyPrincipal?.principalEmail, delegInfo.thirdPartyPrincipal?.principalEmail - }); + ]); } ); } From 0e0f93884b313e6ad1bbb49a2ab580f6d9dfaf87 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Tue, 26 Nov 2024 16:07:14 +0100 Subject: [PATCH 6/6] apply review remarks co-authored by: @efd6 --- ...cemanager-googleapis-com.log-expected.json | 15 +- ...t-compute-googleapis-com.log-expected.json | 3 +- .../elasticsearch/ingest_pipeline/default.yml | 130 ++++++++---------- 3 files changed, 64 insertions(+), 84 deletions(-) diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log-expected.json b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log-expected.json index 0cb8e9d45a2..2987836a147 100644 --- a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log-expected.json +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log-expected.json @@ -143,24 +143,25 @@ "entity": [ "projects/project-id/roles/x", "roles/cloudasset.viewer", - "serviceAccount:member-sa@project-id.iam.gserviceaccount.com", - "serviceAccount:e@project-id.iam.gserviceaccount.com", + "principal://iam.googleapis.com/projects/project-id/locations/global/workloadIdentityPools/...", "serviceAccount:f@project-id.iam.gserviceaccount.com", "serviceAccount:first@project-id.iam.gserviceaccount.com", "projects/project-id/roles/random", + "serviceAccount:b@project-id.iam.gserviceaccount.com", + "serviceAccount:d@project-id.iam.gserviceaccount.com", + "user:doesnotexist@elastic.co", + "projects/project-id/roles/ThatRoleToo", + "serviceAccount:madeupprincipal@project-id.iam.gserviceaccount.com", + "serviceAccount:member-sa@project-id.iam.gserviceaccount.com", + "serviceAccount:e@project-id.iam.gserviceaccount.com", "serviceAccount:a@project-id.iam.gserviceaccount.com", "serviceAccount:second@project-id.iam.gserviceaccount.com", "projects/project-id", "serviceAccount:g@project-id.iam.gserviceaccount.com", "roles/cloudkms.admin", - "serviceAccount:b@project-id.iam.gserviceaccount.com", - "serviceAccount:d@project-id.iam.gserviceaccount.com", "made-up-ci-account@project-id.iam.gserviceaccount.com", "serviceAccount:c@project-id.iam.gserviceaccount.com", - "user:doesnotexist@elastic.co", - "projects/project-id/roles/ThatRoleToo", "projects/project-id/roles/this_role_as_well", - "serviceAccount:madeupprincipal@project-id.iam.gserviceaccount.com", "roles/owner", "group:agroup@elastic.co", "roles/browser" diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log-expected.json b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log-expected.json index 380b316f162..2d874cc183e 100644 --- a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log-expected.json +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log-expected.json @@ -209,8 +209,9 @@ "related": { "entity": [ "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/x-logs-network", - "projects/project-id/zones/us-central1-a/instances/x-logs", "serviceAccount:project-id@cloudservices.gserviceaccount.com", + "projects/project-id/zones/us-central1-a/instances/x-logs", + "principalA@prod.google.com", "project-id@cloudservices.gserviceaccount.com", "x-logs-sa@project-id.iam.gserviceaccount.com" ], diff --git a/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 9c986140faa..5db73156fd8 100644 --- a/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -114,107 +114,85 @@ processors: field: error.message value: "{{{ _ingest.on_failure_message }}}" source: | - void addFields(Set entities, List fields) { - for (String field : fields) { - addField(entities, field); - } - } - - void addField(Set entities, String fieldName) { - addValue(entities, field(fieldName).get(null)); - } - - boolean addValues(Set entities, List values) { - boolean addedAll = true; - for (String value : values) { - addedAll = addedAll && addValue(entities, value); - } - - return addedAll; - } - - boolean addValue(Set entities, String value) { - if (value == null || value == "") { - return false; + void addValue(Set entities, def value) { + if (value != null && value != "") { + entities.add(value); } - - return entities.add(value); } boolean isKubernetes = false; if (ctx.json?.resource?.type != null) { - String resType = ctx.json.resource.type; - if (resType == "k8s_cluster" || resType == "gke_cluster" || resType == "kubernetes") { - isKubernetes = true; - } + String typ = ctx.json.resource.type; + isKubernetes = (typ == "k8s_cluster" || typ == "gke_cluster" || typ == "kubernetes"); } // Using tree set to ensure a sorting is kept (testing purposes) TreeSet entities = new TreeSet(); - addField(entities, "json.protoPayload.request.parent"); + addValue(entities, ctx.json?.protoPayload?.request?.parent); if (!isKubernetes) { - addFields(entities, [ - "json.protoPayload.resourceName", - "json.protoPayload.response.user" - ]); + addValue(entities, ctx.json?.protoPayload?.resourceName); + addValue(entities, ctx.json?.protoPayload?.response?.user); } - - HashMap authInfo = field("json.protoPayload.authenticationInfo").get(new HashMap()); + HashMap authInfo = ctx.json?.protoPayload?.authenticationInfo ?: new HashMap(); if (!isKubernetes) { addValue(entities, authInfo.principalEmail); } - addValues(entities, [ - authInfo.principalSubject, - authInfo.serviceAccountKeyName - ]); - if (authInfo.serviceAccountDelegationInfo != null && !(authInfo.serviceAccountDelegationInfo instanceof List)) { - authInfo.serviceAccountDelegationInfo.get(new ArrayList()).stream().forEach( - delegInfo -> { - addValues(entities, [ - delegInfo.principalSubject, - delegInfo.firstPartyPrincipal?.principalEmail, - delegInfo.thirdPartyPrincipal?.principalEmail - ]); - } - ); + addValue(entities, authInfo.principalSubject); + addValue(entities, authInfo.serviceAccountKeyName); + if (authInfo.serviceAccountDelegationInfo instanceof List) { + for (def i: authInfo.serviceAccountDelegationInfo) { + addValue(entities, i.principalSubject); + addValue(entities, i.firstPartyPrincipal?.principalEmail); + addValue(entities, i.thirdPartyPrincipal?.principalEmail); + } } - String serviceName = field("json.protoPayload.serviceName").get(null); - + String serviceName = ctx.json?.protoPayload?.serviceName ?: ''; if (serviceName == "compute.googleapis.com") { - field("json.protoPayload.request.networkInterfaces").get(new ArrayList()).stream().forEach( - f -> addValue(entities, f.network) - ); - - field("json.protoPayload.request.serviceAccounts").get(new ArrayList()).stream().forEach( - f -> addValue(entities, f.email) - ); - - field("json.protoPayload.request.disks").get(new ArrayList()).stream().forEach( - f -> addValue(entities, f.source) - ); + if (ctx.json?.protoPayload?.request?.networkInterfaces instanceof List) { + for (def e: ctx.json.protoPayload.request.networkInterfaces) { + addValue(entities, e.network); + } + } + if (ctx.json?.protoPayload?.request?.serviceAccounts instanceof List) { + for (def e: ctx.json.protoPayload.request.serviceAccounts) { + addValue(entities, e.email); + } + } + if (ctx.json?.protoPayload?.request?.disks instanceof List) { + for (def e: ctx.json.protoPayload.request.disks) { + addValue(entities, e.source); + } + } } else if (serviceName == "cloudresourcemanager.googleapis.com") { - field("json.protoPayload.request.policy.bindings").get(new ArrayList()).stream().forEach( - f -> { - addValue(entities, f.role); - f.members.forEach(m -> addValue(entities, m)); + if (ctx.json?.protoPayload?.request?.policy?.bindings instanceof List) { + for (def e: ctx.json.protoPayload.request.policy.bindings) { + addValue(entities, e.role); + for (def m: e.members) { + addValue(entities, m); + } } - ); - field("json.protoPayload.response.bindings").get(new ArrayList()).stream().forEach( - f -> { - addValue(entities, f.role); - f.members.forEach(m -> addValue(entities, m)); + } + if (ctx.json?.protoPayload?.response?.bindings instanceof List) { + for (def e: ctx.json.protoPayload.response.bindings) { + addValue(entities, e.role); + for (def m: e.members) { + addValue(entities, m); + } } - ); + } } else if (serviceName == "iamcredentials.googleapis.com") { - field("json.protoPayload.metadata.identityDelegationChain").get(new ArrayList()).stream().forEach( - f -> addValue(entities, f) - ); + if (ctx.json?.protoPayload?.metadata?.identityDelegationChain instanceof List) { + for (def e: ctx.json.protoPayload.metadata.identityDelegationChain) { + addValue(entities, e); + } + } } - field("related.entity").set(entities); + ctx.related = ctx.related ?: [:]; + ctx.related.entity = entities; ## # AuthenticationInfo