feat Adding custom audit log config for each GCP service (#31)

* feat Adding parameter to be able to select a custom audit log config for each service to be ingested

* feat Adding audit log config at the organization level

* feat Adding ingestion synk dynamic exclusions

* feat Updating the documentation for the new inputs

Author: S3B4SZ17

modules/services/webhook-datasource/README.md

@@ -72,6 +72,8 @@ No modules.
 | <a name="input_role_name"></a> [role\_name](#input\_role\_name) | (Optional) Role name for custom role binding to the service account, with read permissions for data ingestion resources | `string` | `"SysdigIngestionAuthRole"` | no |
 | <a name="input_external_id"></a> [external\_id](#input\_external\_id) | (Required) Random string generated unique to a customer | `string` | n/a | yes |
 | <a name="input_suffix"></a> [suffix](#input\_suffix) | (Optional) Suffix to uniquely identify resources during multiple installs. If not provided, random value is autogenerated | `string` | `null` | no |
+| <a name="input_audit_log_config"></a> [audit\_log\_config](#input\_audit\_log\_config) | List of services and their audit log configurations to be ingested. Default is to ingest all logs. | <pre>list(object({<br>    service = string,<br>    log_config = list(object({<br>      log_type         = string,<br>      exempted_members = optional(list(string))<br>    }))<br>  }))</pre> | <pre>[<br>  {<br>    "log_config": [<br>      {<br>        "log_type": "ADMIN_READ"<br>      },<br>      {<br>        "log_type": "DATA_READ"<br>      },<br>      {<br>        "log_type": "DATA_WRITE"<br>      }<br>    ],<br>    "service": "allServices"<br>  }<br>]</pre> | no |
+| <a name="input_exclude_logs_filter"></a> [exclude\_logs\_filter](#input\_exclude\_logs\_filter) | Filter to exclude logs from ingestion. Default is to ingest all google.cloud.audit.AuditLog logs. with no exclusions. | <pre>list(object({<br>    name        = string,<br>    description = optional(string),<br>    filter      = string,<br>    disabled    = optional(bool)<br>  }))</pre> | `[]` | no |
 
 ## Outputs
 

modules/services/webhook-datasource/main.tf

@@ -27,22 +27,28 @@
 # Audit Logs #
 #------------#
 
-resource "google_project_iam_audit_config" "audit_config" {
-  count = var.is_organizational ? 0 : 1
-
-  project = var.project_id
-  service = "allServices"
-
-  audit_log_config {
-    log_type = "ADMIN_READ"
+locals {
+  # Data structure will be a map for each service, that can have multiple audit_log_config
+  audit_log_config = { for audit in var.audit_log_config :
+    audit["service"] => {
+      log_config = audit["log_config"]
+    }
   }
+}
 
-  audit_log_config {
-    log_type = "DATA_READ"
-  }
+resource "google_project_iam_audit_config" "audit_config" {
+  for_each = var.is_organizational ? {} : local.audit_log_config
 
-  audit_log_config {
-    log_type = "DATA_WRITE"
+  project = var.project_id
+  service = each.key
+
+  dynamic "audit_log_config" {
+    for_each = each.value.log_config
+    iterator = log_config
+    content {
+      log_type         = log_config.value.log_type
+      exempted_members = log_config.value.exempted_members
+    }
   }
 }
 
@@ -79,6 +85,17 @@ resource "google_logging_project_sink" "ingestion_sink" {
 
   filter = "protoPayload.@type = \"type.googleapis.com/google.cloud.audit.AuditLog\""
 
+  # Dynamic block to exclude logs from ingestion
+  dynamic "exclusions" {
+    for_each = var.exclude_logs_filter
+    content {
+      name        = exclusions.value.name
+      description = exclusions.value.description
+      filter      = exclusions.value.filter
+      disabled    = exclusions.value.disabled
+    }
+  }
+
   # NOTE: Used to create a dedicated writer identity and not using the default one
   unique_writer_identity = true
 }

modules/services/webhook-datasource/organizational.tf

@@ -11,21 +11,18 @@ data "google_organization" "org" {
 # Audit Logs #
 #------------#
 resource "google_organization_iam_audit_config" "audit_config" {
-  count = var.is_organizational ? 1 : 0
+  for_each = var.is_organizational ? local.audit_log_config : {}
 
   org_id  = data.google_organization.org[0].org_id
-  service = "allServices"
-
-  audit_log_config {
-    log_type = "ADMIN_READ"
-  }
+  service = each.key
 
-  audit_log_config {
-    log_type = "DATA_READ"
-  }
-
-  audit_log_config {
-    log_type = "DATA_WRITE"
+  dynamic "audit_log_config" {
+    for_each = each.value.log_config
+    iterator = log_config
+    content {
+      log_type         = log_config.value.log_type
+      exempted_members = log_config.value.exempted_members
+    }
   }
 }
 
@@ -44,6 +41,17 @@ resource "google_logging_organization_sink" "ingestion_sink" {
   destination = "pubsub.googleapis.com/projects/${var.project_id}/topics/${google_pubsub_topic.ingestion_topic.name}"
   filter      = "protoPayload.@type = \"type.googleapis.com/google.cloud.audit.AuditLog\""
 
+  # Dynamic block to exclude logs from ingestion
+  dynamic "exclusions" {
+    for_each = var.exclude_logs_filter
+    content {
+      name        = exclusions.value.name
+      description = exclusions.value.description
+      filter      = exclusions.value.filter
+      disabled    = exclusions.value.disabled
+    }
+  }
+
   # NOTE: The include_children attribute is set to true in order to ingest data
   # even from potential sub-organizations
   include_children = true

modules/services/webhook-datasource/variables.tf

@@ -74,3 +74,35 @@ variable "suffix" {
   description = "Suffix to uniquely identify resources during multiple installs. If not provided, random value is autogenerated"
   default     = null
 }
+
+variable "audit_log_config" {
+  description = "List of services and their audit log configurations to be ingested. Default is to ingest all logs."
+  type = list(object({
+    service = string,
+    log_config = list(object({
+      log_type         = string,
+      exempted_members = optional(list(string))
+    }))
+  }))
+  default = [
+    {
+      service = "allServices"
+      log_config = [
+        { log_type = "ADMIN_READ" },
+        { log_type = "DATA_READ" },
+        { log_type = "DATA_WRITE" }
+      ]
+    }
+  ]
+}
+
+variable "exclude_logs_filter" {
+  description = "Filter to exclude logs from ingestion. Default is to ingest all google.cloud.audit.AuditLog logs. with no exclusions."
+  type = list(object({
+    name        = string,
+    description = optional(string),
+    filter      = string,
+    disabled    = optional(bool)
+  }))
+  default = []
+}

test/examples/secure_threat_detection/organization/main.tf

@@ -4,12 +4,45 @@ provider "google" {
 }
 
 module "organization-threat-detection" {
-  source            	= "../../../..//modules/services/webhook-datasource"
-  project_id        	= "mytestproject"
-  push_endpoint     	= "test_sysdig_secure_cloudingestion_endpoint"
-  is_organizational 	= true
+  source              = "../../../..//modules/services/webhook-datasource"
+  project_id          = "mytestproject"
+  push_endpoint       = "test_sysdig_secure_cloudingestion_endpoint"
+  is_organizational   = true
   organization_domain = "mytestorg.com"
   external_id         = "external_id"
+  audit_log_config = [
+    {
+      service = "cloudsql.googleapis.com"
+      log_config = [{ log_type = "DATA_READ",
+        exempted_members = [
+          "serviceAccount:[email protected]",
+        ]
+        },
+        { log_type = "DATA_WRITE" }
+      ]
+    },
+    {
+      service = "storage.googleapis.com"
+      log_config = [{ log_type = "DATA_WRITE"
+      }]
+    },
+    {
+      service    = "container.googleapis.com"
+      log_config = [{ log_type = "DATA_READ" }]
+    }
+  ]
+  exclude_logs_filter = [
+    {
+      name        = "nsexcllusion2"
+      description = "Exclude logs from namespace-2 in k8s"
+      filter      = "resource.type = k8s_container resource.labels.namespace_name=\"namespace-2\" "
+    },
+    {
+      name        = "nsexcllusion1"
+      description = "Exclude logs from namespace-1 in k8s"
+      filter      = "resource.type = k8s_container resource.labels.namespace_name=\"namespace-1\" "
+    }
+  ]
 }
 
 module "organization-posture" {

test/examples/secure_threat_detection/single/main.tf

@@ -8,6 +8,39 @@ module "single-project-threat-detection" {
   project_id    = "mytestproject"
   push_endpoint = "test_sysdig_secure_cloudingestion_endpoint"
   external_id   = "external_id"
+  audit_log_config = [
+    {
+      service = "cloudsql.googleapis.com"
+      log_config = [{ log_type = "DATA_READ",
+        exempted_members = [
+          "serviceAccount:[email protected]",
+        ]
+        },
+        { log_type = "DATA_WRITE" }
+      ]
+    },
+    {
+      service = "storage.googleapis.com"
+      log_config = [{ log_type = "DATA_WRITE"
+      }]
+    },
+    {
+      service    = "container.googleapis.com"
+      log_config = [{ log_type = "DATA_READ" }]
+    }
+  ]
+  exclude_logs_filter = [
+    {
+      name        = "nsexcllusion2"
+      description = "Exclude logs from namespace-2 in k8s"
+      filter      = "resource.type = k8s_container resource.labels.namespace_name=\"namespace-2\" "
+    },
+    {
+      name        = "nsexcllusion1"
+      description = "Exclude logs from namespace-1 in k8s"
+      filter      = "resource.type = k8s_container resource.labels.namespace_name=\"namespace-1\" "
+    }
+  ]
 }
 
 terraform {
@@ -66,4 +99,3 @@ resource "sysdig_secure_cloud_auth_account" "gcp_project_mytestproject" {
     })
   }
 }
-