fix: adjust log filters for SHA/CIS compliance (#261)

Author: amandakarina

1-org/README.md

@@ -15,6 +15,10 @@ Enabling Data Access logs might result in your project being charged for the add
 For details on costs you might incur, go to [Pricing](https://cloud.google.com/stackdriver/pricing).
 You can choose not to enable the Data Access logs by setting variable `data_access_logs_enabled` to false.
 
+**Note:** This module creates sink to export all logs to Google Storage. It's also create sinks to export a subset of security related logs
+to Bigquery and Pub/Sub. This will result in additional charges for those copies of logs.
+You can change the filters & sinks by modifying the configuration in `envs/shared/log_sinks.tf`.
+
 **Note:** Currently, this module does not enable bucket policy retention for organization logs, please, enable it if needed.
 
 ### Setup to run via Cloud Build

1-org/envs/shared/README.md

@@ -18,6 +18,7 @@
 | interconnect\_project\_alert\_pubsub\_topic | The name of the Cloud Pub/Sub topic where budget related messages will be published, in the form of `projects/{project_id}/topics/{topic_id}` for the interconnect project. | string | `"null"` | no |
 | interconnect\_project\_alert\_spent\_percents | A list of percentages of the budget to alert on when threshold is exceeded for the interconnect project. | list(number) | `<list>` | no |
 | interconnect\_project\_budget\_amount | The amount to use as the budget for the interconnect project. | number | `"1000"` | no |
+| log\_export\_storage\_force\_destroy | (Optional) If set to true, delete all contents when destroying the resource; otherwise, destroying the resource will fail if contents are present. | bool | `"false"` | no |
 | log\_export\_storage\_location | The location of the storage bucket used to export logs. | string | `"US"` | no |
 | log\_export\_storage\_retention\_policy | Configuration of the bucket's data retention policy for how long objects in the bucket should be retained. | object | `"null"` | no |
 | org\_audit\_logs\_project\_alert\_pubsub\_topic | The name of the Cloud Pub/Sub topic where budget related messages will be published, in the form of `projects/{project_id}/topics/{topic_id}` for the org audit logs project. | string | `"null"` | no |

1-org/envs/shared/log_sinks.tf

@@ -17,14 +17,15 @@
 locals {
   parent_resource_id   = var.parent_folder != "" ? var.parent_folder : var.org_id
   parent_resource_type = var.parent_folder != "" ? "folder" : "organization"
-  all_logs_filter      = <<EOF
+  main_logs_filter     = <<EOF
     logName: /logs/cloudaudit.googleapis.com%2Factivity OR
     logName: /logs/cloudaudit.googleapis.com%2Fsystem_event OR
     logName: /logs/cloudaudit.googleapis.com%2Fdata_access OR
     logName: /logs/compute.googleapis.com%2Fvpc_flows OR
     logName: /logs/compute.googleapis.com%2Ffirewall OR
     logName: /logs/cloudaudit.googleapis.com%2Faccess_transparency
 EOF
+  all_logs_filter      = ""
 }
 
 resource "random_string" "suffix" {
@@ -41,7 +42,7 @@ module "log_export_to_biqquery" {
   source                 = "terraform-google-modules/log-export/google"
   version                = "~> 5.0"
   destination_uri        = module.bigquery_destination.destination_uri
-  filter                 = local.all_logs_filter
+  filter                 = local.main_logs_filter
   log_sink_name          = "sk-c-logging-bq"
   parent_resource_id     = local.parent_resource_id
   parent_resource_type   = local.parent_resource_type
@@ -84,6 +85,7 @@ module "storage_destination" {
   uniform_bucket_level_access = true
   location                    = var.log_export_storage_location
   retention_policy            = var.log_export_storage_retention_policy
+  force_destroy               = var.log_export_storage_force_destroy
 }
 
 /******************************************
@@ -94,7 +96,7 @@ module "log_export_to_pubsub" {
   source                 = "terraform-google-modules/log-export/google"
   version                = "~> 5.0"
   destination_uri        = module.pubsub_destination.destination_uri
-  filter                 = local.all_logs_filter
+  filter                 = local.main_logs_filter
   log_sink_name          = "sk-c-logging-pub"
   parent_resource_id     = local.parent_resource_id
   parent_resource_type   = local.parent_resource_type

1-org/envs/shared/variables.tf

@@ -96,6 +96,12 @@ variable "log_export_storage_location" {
   default     = "US"
 }
 
+variable "log_export_storage_force_destroy" {
+  description = "(Optional) If set to true, delete all contents when destroying the resource; otherwise, destroying the resource will fail if contents are present."
+  type        = bool
+  default     = false
+}
+
 variable "audit_logs_table_delete_contents_on_destroy" {
   description = "(Optional) If set to true, delete all the tables in the dataset when destroying the resource; otherwise, destroying the resource will fail if tables are present."
   type        = bool

test/fixtures/org/main.tf

@@ -32,4 +32,5 @@ module "test" {
   domains_to_allow                            = [var.domain_to_allow]
   create_access_context_manager_access_policy = false
   audit_logs_table_delete_contents_on_destroy = true
+  log_export_storage_force_destroy            = true
 }

test/integration/org/controls/gcp_logging.rb

@@ -17,8 +17,7 @@
 parent_resource_id = attribute('parent_resource_id')
 logs_export_storage_bucket_name = attribute('logs_export_storage_bucket_name')
 logs_export_pubsub_topic = attribute('logs_export_pubsub_topic')
-
-all_logs_filter = [
+main_logs_filter = [
   'logName: /logs/cloudaudit.googleapis.com%2Factivity',
   'logName: /logs/cloudaudit.googleapis.com%2Fsystem_event',
   'logName: /logs/cloudaudit.googleapis.com%2Fdata_access',
@@ -62,7 +61,7 @@
     name: 'sk-c-logging-bq'
   ) do
     it { should exist }
-    all_logs_filter.each do |filter|
+    main_logs_filter.each do |filter|
       its('filter') do
         should include filter
       end
@@ -76,10 +75,8 @@
     name: 'sk-c-logging-bkt'
   ) do
     it { should exist }
-    all_logs_filter.each do |filter|
-      its('filter') do
-        should include filter
-      end
+    its('filter') do
+      should be_nil
     end
     its('include_children') { should cmp 'true' }
     its('destination') { should cmp "storage.googleapis.com/#{logs_export_storage_bucket_name}" }
@@ -90,7 +87,7 @@
     name: 'sk-c-logging-pub'
   ) do
     it { should exist }
-    all_logs_filter.each do |filter|
+    main_logs_filter.each do |filter|
       its('filter') do
         should include filter
       end