From 0f2b0c8e110be68c6049d4161120bb8e1a4e2275 Mon Sep 17 00:00:00 2001 From: Jose Pablo Camacho Date: Tue, 13 May 2025 16:10:32 -0600 Subject: [PATCH 1/3] SSPROD-56045 - fix gcp api setup --- .../cdr_ciem/main.tf | 39 ----- .../organization_api_enablement/cspm/main.tf | 157 ------------------ .../organization_api_enablement/vm/main.tf | 142 ---------------- .../single_api_enablement/cdr_ciem/main.tf | 2 +- .../single_api_enablement/cspm/main.tf | 2 +- .../examples/single_api_enablement/vm/main.tf | 2 +- 6 files changed, 3 insertions(+), 341 deletions(-) delete mode 100644 test/examples/organization_api_enablement/cdr_ciem/main.tf delete mode 100644 test/examples/organization_api_enablement/cspm/main.tf delete mode 100644 test/examples/organization_api_enablement/vm/main.tf diff --git a/test/examples/organization_api_enablement/cdr_ciem/main.tf b/test/examples/organization_api_enablement/cdr_ciem/main.tf deleted file mode 100644 index eb74b40..0000000 --- a/test/examples/organization_api_enablement/cdr_ciem/main.tf +++ /dev/null @@ -1,39 +0,0 @@ -/* -This terraform file is intended to enable the GCP APIs needed for CDR/CIEM feature within a GCP organization onboarding. -It will create a google_project_service resource per each service enabled within each GCP project. -The APIs needed for the CDR/CIEM feature are listed below: - - Cloud Pub/Sub API - -* Note: This do not overwrite any other APIs config that your GCP project has, it will only enabled it if isn't yet. -*/ - -# Set local local variables for Project ID and API services to enable -locals { - project = "org-child-project-1" - services = [ - "pubsub.googleapis.com" - ] -} - -# GCP provider -provider "google" { - project = local.project - region = "us-west-1" -} - -// Enable API services for GCP project -resource "google_project_service" "enable_cdr_ciem_apis" { - project = local.project - - for_each = toset(local.services) - service = each.value - disable_on_destroy = false -} - -# Output the projects and APIs enabled -output "enabled_projects" { - value = distinct([for service in local.services : google_project_service.enable_cdr_ciem_apis[service].project]) -} -output "enabled_services" { - value = [for service in local.services : google_project_service.enable_cdr_ciem_apis[service].service] -} \ No newline at end of file diff --git a/test/examples/organization_api_enablement/cspm/main.tf b/test/examples/organization_api_enablement/cspm/main.tf deleted file mode 100644 index 168b862..0000000 --- a/test/examples/organization_api_enablement/cspm/main.tf +++ /dev/null @@ -1,157 +0,0 @@ -/* -This terraform file is intended to enable the GCP APIs needed for CSPM feature within a GCP organization onboarding. -It will create a google_project_service resource per each service enabled within each GCP project. -The APIs needed for the CSPM feature are listed below: - - Security Token Service API - - Cloud Asset API - - Cloud Identity API - - Admin SDK API -In addition, since CSPM is needed for onboard any GCP project these other APIs are also enabled: - - Identity and access management API - - IAM Service Account Credentials API - - Cloud Resource Manager API - -* Note: This do not overwrite any other APIs config that your GCP project has, it will only enabled it if isn't yet. -*/ - -# Set local variables for Organization ID and API services to enable -locals { - services = [ - # CSPM specific APIs - "sts.googleapis.com", - "cloudasset.googleapis.com", - "cloudidentity.googleapis.com", - "admin.googleapis.com", - - # additional APIs - "iam.googleapis.com", - "iamcredentials.googleapis.com", - "cloudresourcemanager.googleapis.com" - ] - root_projects = [for project in data.google_projects.organization_projects.projects : project.project_id] - folder_projects = jsondecode(data.local_file.projects_from_folder.content) - all_projects = concat(local.root_projects, local.folder_projects) - project_and_services = flatten([ - for project in local.all_projects : [ - for service in local.services : { - project = project - service = service - } - ] - ]) -} - -# GCP provider -provider "google" { - region = "us-west-1" -} - -# Get list of projects under the specified organization -data "google_projects" "organization_projects" { - filter = "parent.type:organization parent.id:${data.google_organization.org.org_id}" -} - -data "google_organization" "org" { - domain = "draios.com" -} - -data "local_file" "projects_from_folder" { - filename = "project_ids.json" - depends_on = [null_resource.get_projects_from_folders] -} - -# Enable API services for GCP project -resource "google_project_service" "enable_cspm_apis" { - // create a unique key per project and service to enable each API - for_each = { for item in local.project_and_services : "${item.project}-${item.service}" => item } - - project = each.value.project - service = each.value.service - disable_on_destroy = false -} - -# Output the projects and APIs enabled -output "enabled_projects" { - value = distinct([for resource in google_project_service.enable_cspm_apis : resource.project]) -} - -output "enabled_services" { - value = distinct([for service in google_project_service.enable_cspm_apis : service.service]) -} - -# Script to get projects from folders recursively and set to a file -resource "null_resource" "get_projects_from_folders" { - provisioner "local-exec" { - interpreter = ["/bin/bash", "-c"] - command = </dev/null 2>&1; then - echo "Invalid JSON returned for projects list." - return - fi - - # get project ids - local project_ids=$(echo "$projects_json" | jq -r '.[] | .projectId') - - # check project ids not empty and add to global variable - if [ -n "$project_ids" ]; then - for project_id in $project_ids; do - FINAL_PROJECT_IDS+=("$project_id") - done - else - echo "No projects found in folder $folder_id" - fi - } - - list_folders_recursive() { - local parent_id=$1 - local parent_type=$2 - - # list folders on org or other folders - if [[ "$parent_type" == "organization" ]]; then - folders=$(gcloud resource-manager folders list --organization=$parent_id --format=json) - elif [[ "$parent_type" == "folder" ]]; then - folders=$(gcloud resource-manager folders list --folder=$parent_id --format=json) - fi - - # check if there were folders returned - if [ "$(echo "$folders" | jq length)" -eq 0 ]; then - return - fi - - # iterate over folder and call functions recursively - for folder in $(echo "$folders" | jq -c '.[]'); do - folder_id=$(echo "$folder" | jq -r '.name' | awk -F'/' '{print $NF}') - - list_projects "$folder_id" - list_folders_recursive "$folder_id" "folder" - done - } - - # start organization scraping - ORG_JSON=$(gcloud organizations list --filter="displayName:$ORG_DOMAIN" --format=json) - ORG_ID=$(echo "$ORG_JSON" | jq -r '.[0].name' | sed 's/organizations\///') - if [ -z "$ORG_ID" ]; then - echo "Organization with display name '$DISPLAY_NAME' not found." - exit 1 - fi - - echo "Listing all projects in folders for organization: $ORG_DOMAIN" - list_folders_recursive "$ORG_ID" "organization" - printf "%s\n" "$${FINAL_PROJECT_IDS[@]}" | jq -R . | jq -s . > "project_ids.json" - echo "Projects listed and saved to local file." - EOF - } -} \ No newline at end of file diff --git a/test/examples/organization_api_enablement/vm/main.tf b/test/examples/organization_api_enablement/vm/main.tf deleted file mode 100644 index e4c0957..0000000 --- a/test/examples/organization_api_enablement/vm/main.tf +++ /dev/null @@ -1,142 +0,0 @@ -/* -This terraform file is intended to enable the GCP APIs needed for VM feature within a GCP organization onboarding. -It will create a google_project_service resource per each service enabled within each GCP project. -The APIs needed for the VM feature are listed below: - - Compute Engine API - -* Note: This do not overwrite any other APIs config that your GCP project has, it will only enabled it if isn't yet. -*/ - -# Set local variables for Organization ID and API services to enable -locals { - services = [ - "compute.googleapis.com" - ] - root_projects = [for project in data.google_projects.organization_projects.projects : project.project_id] - folder_projects = jsondecode(data.local_file.projects_from_folder.content) - all_projects = concat(local.root_projects, local.folder_projects) - project_and_services = flatten([ - for project in local.all_projects : [ - for service in local.services : { - project = project - service = service - } - ] - ]) -} - -# GCP provider -provider "google" { - region = "us-west-1" -} - -# Get list of projects under the specified organization -data "google_projects" "organization_projects" { - filter = "parent.type:organization parent.id:${data.google_organization.org.org_id}" -} - -data "google_organization" "org" { - domain = "draios.com" -} - -data "local_file" "projects_from_folder" { - filename = "project_ids.json" - depends_on = [null_resource.get_projects_from_folders] -} - -# Enable API services for GCP project -resource "google_project_service" "enable_vm_apis" { - // create a unique key per project and service to enable each API - for_each = { for item in local.project_and_services : "${item.project}-${item.service}" => item } - - project = each.value.project - service = each.value.service - disable_on_destroy = false -} - -# Output the projects and APIs enabled -output "enabled_projects" { - value = distinct([for resource in google_project_service.enable_vm_apis : resource.project]) -} - -output "enabled_services" { - value = distinct([for service in google_project_service.enable_vm_apis : service.service]) -} - - -# Script to get projects from folders recursively and set to a file -resource "null_resource" "get_projects_from_folders" { - provisioner "local-exec" { - interpreter = ["/bin/bash", "-c"] - command = </dev/null 2>&1; then - echo "Invalid JSON returned for projects list." - return - fi - - # get project ids - local project_ids=$(echo "$projects_json" | jq -r '.[] | .projectId') - - # check project ids not empty and add to global variable - if [ -n "$project_ids" ]; then - for project_id in $project_ids; do - FINAL_PROJECT_IDS+=("$project_id") - done - else - echo "No projects found in folder $folder_id" - fi - } - - list_folders_recursive() { - local parent_id=$1 - local parent_type=$2 - - # list folders on org or other folders - if [[ "$parent_type" == "organization" ]]; then - folders=$(gcloud resource-manager folders list --organization=$parent_id --format=json) - elif [[ "$parent_type" == "folder" ]]; then - folders=$(gcloud resource-manager folders list --folder=$parent_id --format=json) - fi - - # check if there were folders returned - if [ "$(echo "$folders" | jq length)" -eq 0 ]; then - return - fi - - # iterate over folder and call functions recursively - for folder in $(echo "$folders" | jq -c '.[]'); do - folder_id=$(echo "$folder" | jq -r '.name' | awk -F'/' '{print $NF}') - - list_projects "$folder_id" - list_folders_recursive "$folder_id" "folder" - done - } - - # start organization scraping - ORG_JSON=$(gcloud organizations list --filter="displayName:$ORG_DOMAIN" --format=json) - ORG_ID=$(echo "$ORG_JSON" | jq -r '.[0].name' | sed 's/organizations\///') - if [ -z "$ORG_ID" ]; then - echo "Organization with display name '$DISPLAY_NAME' not found." - exit 1 - fi - - echo "Listing all projects in folders for organization: $ORG_ID" - list_folders_recursive "$ORG_ID" "organization" - printf "%s\n" "$${FINAL_PROJECT_IDS[@]}" | jq -R . | jq -s . > "project_ids.json" - echo "Projects listed and saved to local file." - EOF - } -} \ No newline at end of file diff --git a/test/examples/single_api_enablement/cdr_ciem/main.tf b/test/examples/single_api_enablement/cdr_ciem/main.tf index a9421a3..a949f75 100644 --- a/test/examples/single_api_enablement/cdr_ciem/main.tf +++ b/test/examples/single_api_enablement/cdr_ciem/main.tf @@ -9,7 +9,7 @@ The APIs needed for the CDR/CIEM feature are listed below: # Set local local variables for Project ID and API services to enable locals { - project = "org-child-project-1" + project = "" services = [ "pubsub.googleapis.com" ] diff --git a/test/examples/single_api_enablement/cspm/main.tf b/test/examples/single_api_enablement/cspm/main.tf index b09d420..0067f13 100644 --- a/test/examples/single_api_enablement/cspm/main.tf +++ b/test/examples/single_api_enablement/cspm/main.tf @@ -16,7 +16,7 @@ In addition, since CSPM is needed for onboard any GCP project these other APIs a # Set local variables for Project ID and API services to enable locals { - project = "org-child-project-1" + project = "" services = [ # CSPM specific APIs "sts.googleapis.com", diff --git a/test/examples/single_api_enablement/vm/main.tf b/test/examples/single_api_enablement/vm/main.tf index 961c9c6..f64a55a 100644 --- a/test/examples/single_api_enablement/vm/main.tf +++ b/test/examples/single_api_enablement/vm/main.tf @@ -9,7 +9,7 @@ The APIs needed for the VM feature are listed below: # Set local variables for Project ID and API services to enable locals { - project = "org-child-project-1" + project = "" services = [ "compute.googleapis.com" ] From f2a0278efd04efa6c8b088cc0fdd563df766dd19 Mon Sep 17 00:00:00 2001 From: Jose Pablo Camacho Date: Tue, 13 May 2025 16:12:09 -0600 Subject: [PATCH 2/3] SSPROD-56045 - fix gcp api setup --- .../cdr_ciem/main.tf | 0 .../cspm/main.tf | 0 .../{single_api_enablement => project_api_enablement}/vm/main.tf | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename test/examples/{single_api_enablement => project_api_enablement}/cdr_ciem/main.tf (100%) rename test/examples/{single_api_enablement => project_api_enablement}/cspm/main.tf (100%) rename test/examples/{single_api_enablement => project_api_enablement}/vm/main.tf (100%) diff --git a/test/examples/single_api_enablement/cdr_ciem/main.tf b/test/examples/project_api_enablement/cdr_ciem/main.tf similarity index 100% rename from test/examples/single_api_enablement/cdr_ciem/main.tf rename to test/examples/project_api_enablement/cdr_ciem/main.tf diff --git a/test/examples/single_api_enablement/cspm/main.tf b/test/examples/project_api_enablement/cspm/main.tf similarity index 100% rename from test/examples/single_api_enablement/cspm/main.tf rename to test/examples/project_api_enablement/cspm/main.tf diff --git a/test/examples/single_api_enablement/vm/main.tf b/test/examples/project_api_enablement/vm/main.tf similarity index 100% rename from test/examples/single_api_enablement/vm/main.tf rename to test/examples/project_api_enablement/vm/main.tf From 241344ed1e8458579970049dc1f74465e0a52e8c Mon Sep 17 00:00:00 2001 From: Jose Pablo Camacho Date: Tue, 20 May 2025 14:15:11 -0600 Subject: [PATCH 3/3] SSPROD-56045 - fix gcp api setup --- test/examples/project_api_enablement/cdr_ciem/main.tf | 4 +++- test/examples/project_api_enablement/cspm/main.tf | 4 ---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/test/examples/project_api_enablement/cdr_ciem/main.tf b/test/examples/project_api_enablement/cdr_ciem/main.tf index a949f75..9fa0d9b 100644 --- a/test/examples/project_api_enablement/cdr_ciem/main.tf +++ b/test/examples/project_api_enablement/cdr_ciem/main.tf @@ -3,6 +3,7 @@ This terraform file is intended to enable the GCP APIs needed for CDR/CIEM featu It will create a google_project_service resource per each service enabled within the GCP project. The APIs needed for the CDR/CIEM feature are listed below: - Cloud Pub/Sub API + - Cloud Logging API * Note: This do not overwrite any other APIs config that your GCP project has, it will only enabled it if isn't yet. */ @@ -11,7 +12,8 @@ The APIs needed for the CDR/CIEM feature are listed below: locals { project = "" services = [ - "pubsub.googleapis.com" + "pubsub.googleapis.com", + "logging.googleapis.com" ] } diff --git a/test/examples/project_api_enablement/cspm/main.tf b/test/examples/project_api_enablement/cspm/main.tf index 0067f13..79c0c3a 100644 --- a/test/examples/project_api_enablement/cspm/main.tf +++ b/test/examples/project_api_enablement/cspm/main.tf @@ -2,9 +2,7 @@ This terraform file is intended to enable the GCP APIs needed for CSPM feature within a single project onboarding. It will create a google_project_service resource per each service enabled within the GCP project. The APIs needed for the CSPM feature are listed below: - - Security Token Service API - Cloud Asset API - - Cloud Identity API - Admin SDK API In addition, since CSPM is needed for onboard any GCP project these other APIs are also enabled: - Identity and access management API @@ -19,9 +17,7 @@ locals { project = "" services = [ # CSPM specific APIs - "sts.googleapis.com", "cloudasset.googleapis.com", - "cloudidentity.googleapis.com", "admin.googleapis.com", # additional APIs