fix: update bucket naming to comply with guide definition (#904)

daniel-cit · bharathkkb · web-flow · commit 49347f57ab50 · 2022-12-15T17:59:53.000-06:00
Co-authored-by: Bharath KKB &lt;bharathkrishnakb@gmail.com&gt;
diff --git a/0-bootstrap/README.md b/0-bootstrap/README.md
@@ -308,6 +308,7 @@ Each step has instructions for this change.
 | csr\_repos | List of Cloud Source Repos created by the module, linked to Cloud Build triggers. |
 | environment\_step\_terraform\_service\_account\_email | Environment Step Terraform Account |
 | gcs\_bucket\_cloudbuild\_artifacts | Bucket used to store Cloud/Build artifacts in CloudBuild project. |
+| gcs\_bucket\_cloudbuild\_logs | Bucket used to store Cloud/Build logs in CloudBuild project. |
 | gcs\_bucket\_tfstate | Bucket used for storing terraform state for Foundations Pipelines in Seed Project. |
 | group\_billing\_admins | Google Group for GCP Billing Administrators. |
 | group\_org\_admins | Google Group for GCP Organization Administrators. |
diff --git a/0-bootstrap/cb.tf b/0-bootstrap/cb.tf
@@ -69,15 +69,15 @@ module "gcp_projects_state_bucket" {
   source  = "terraform-google-modules/cloud-storage/google//modules/simple_bucket"
   version = "~> 3.2"
 
-  name          = "bkt-b-gcp-projects-tfstate-${module.seed_bootstrap.seed_project_id}"
+  name          = "${var.bucket_prefix}-${module.seed_bootstrap.seed_project_id}-gcp-projects-tfstate"
   project_id    = module.seed_bootstrap.seed_project_id
   location      = var.default_region
   force_destroy = var.bucket_force_destroy
 }
 
 module "tf_source" {
   source  = "terraform-google-modules/bootstrap/google//modules/tf_cloudbuild_source"
-  version = "~> 6.3"
+  version = "~> 6.4"
 
   org_id                = var.org_id
   folder_id             = google_folder.bootstrap.id
@@ -144,7 +144,7 @@ module "tf_private_pool" {
 
 module "tf_cloud_builder" {
   source  = "terraform-google-modules/bootstrap/google//modules/tf_cloudbuild_builder"
-  version = "~> 6.3"
+  version = "~> 6.4"
 
   project_id                   = module.tf_source.cloudbuild_project_id
   dockerfile_repo_uri          = module.tf_source.csr_repos[local.cloudbuilder_repo].url
@@ -155,6 +155,7 @@ module "tf_cloud_builder" {
   trigger_location             = var.default_region
   enable_worker_pool           = true
   worker_pool_id               = module.tf_private_pool.private_worker_pool_id
+  bucket_name                  = "${var.bucket_prefix}-${module.tf_source.cloudbuild_project_id}-tf-cloudbuilder-build-logs"
 }
 
 module "bootstrap_csr_repo" {
@@ -193,7 +194,7 @@ module "build_terraform_image" {
 
 module "tf_workspace" {
   source   = "terraform-google-modules/bootstrap/google//modules/tf_cloudbuild_workspace"
-  version  = "~> 6.3"
+  version  = "~> 6.4"
   for_each = local.granular_sa
 
   project_id                = module.tf_source.cloudbuild_project_id
@@ -202,6 +203,8 @@ module "tf_workspace" {
   enable_worker_pool        = true
   worker_pool_id            = module.tf_private_pool.private_worker_pool_id
   state_bucket_self_link    = local.cb_config[each.key].state_bucket
+  log_bucket_name           = "${var.bucket_prefix}-${module.tf_source.cloudbuild_project_id}-${local.cb_config[each.key].source}-build-logs"
+  artifacts_bucket_name     = "${var.bucket_prefix}-${module.tf_source.cloudbuild_project_id}-${local.cb_config[each.key].source}-build-artifacts"
   cloudbuild_plan_filename  = "cloudbuild-tf-plan.yaml"
   cloudbuild_apply_filename = "cloudbuild-tf-apply.yaml"
   tf_repo_uri               = module.tf_source.csr_repos[local.cb_config[each.key].source].url
diff --git a/0-bootstrap/main.tf b/0-bootstrap/main.tf
@@ -50,7 +50,7 @@ module "seed_bootstrap" {
   org_id                         = var.org_id
   folder_id                      = google_folder.bootstrap.id
   project_id                     = "${var.project_prefix}-b-seed"
-  state_bucket_name              = "${var.bucket_prefix}-b-tfstate"
+  state_bucket_name              = "${var.bucket_prefix}-${var.project_prefix}-b-seed-tfstate"
   force_destroy                  = var.bucket_force_destroy
   billing_account                = var.billing_account
   group_org_admins               = local.group_org_admins
diff --git a/0-bootstrap/outputs.tf b/0-bootstrap/outputs.tf
@@ -97,6 +97,11 @@ output "gcs_bucket_cloudbuild_artifacts" {
   value       = { for key, value in module.tf_workspace : key => replace(value.artifacts_bucket, local.bucket_self_link_prefix, "") }
 }
 
+output "gcs_bucket_cloudbuild_logs" {
+  description = "Bucket used to store Cloud/Build logs in CloudBuild project."
+  value       = { for key, value in module.tf_workspace : key => replace(value.logs_bucket, local.bucket_self_link_prefix, "") }
+}
+
 output "projects_gcs_bucket_tfstate" {
   description = "Bucket used for storing terraform state for stage 4-projects foundations pipelines in seed project."
   value       = module.gcp_projects_state_bucket.bucket.name
diff --git a/4-projects/business_unit_1/shared/README.md b/4-projects/business_unit_1/shared/README.md
@@ -16,6 +16,7 @@
 | cloudbuild\_project\_id | n/a |
 | default\_region | Default region to create resources where applicable. |
 | enable\_cloudbuild\_deploy | Enable infra deployment using Cloud Build. |
+| log\_buckets | GCS Buckets to store Cloud Build logs |
 | plan\_triggers\_id | CB plan triggers |
 | repos | CSRs to store source code |
 | state\_buckets | GCS Buckets to store TF state |
diff --git a/4-projects/business_unit_1/shared/outputs.tf b/4-projects/business_unit_1/shared/outputs.tf
@@ -43,6 +43,11 @@ output "state_buckets" {
   value       = try(module.infra_pipelines[0].state_buckets, {})
 }
 
+output "log_buckets" {
+  description = "GCS Buckets to store Cloud Build logs"
+  value       = try(module.infra_pipelines[0].log_buckets, {})
+}
+
 output "plan_triggers_id" {
   description = "CB plan triggers"
   value       = try(module.infra_pipelines[0].plan_triggers_id, [])
diff --git a/4-projects/business_unit_2/shared/README.md b/4-projects/business_unit_2/shared/README.md
@@ -16,6 +16,7 @@
 | cloudbuild\_project\_id | n/a |
 | default\_region | Default region to create resources where applicable. |
 | enable\_cloudbuild\_deploy | Enable infra deployment using Cloud Build. |
+| log\_buckets | GCS Buckets to store Cloud Build logs |
 | plan\_triggers\_id | CB plan triggers |
 | repos | CSRs to store source code |
 | state\_buckets | GCS Buckets to store TF state |
diff --git a/4-projects/business_unit_2/shared/outputs.tf b/4-projects/business_unit_2/shared/outputs.tf
@@ -43,6 +43,11 @@ output "state_buckets" {
   value       = try(module.infra_pipelines[0].state_buckets, {})
 }
 
+output "log_buckets" {
+  description = "GCS Buckets to store Cloud Build logs"
+  value       = try(module.infra_pipelines[0].log_buckets, {})
+}
+
 output "plan_triggers_id" {
   description = "CB plan triggers"
   value       = try(module.infra_pipelines[0].plan_triggers_id, [])
diff --git a/4-projects/modules/base_env/README.md b/4-projects/modules/base_env/README.md
@@ -7,7 +7,7 @@
 | business\_unit | The business (ex. business\_unit\_1). | `string` | n/a | yes |
 | env | The environment to prepare (ex. development). | `string` | n/a | yes |
 | firewall\_enable\_logging | Toggle firewall logging for VPC Firewalls. | `bool` | `true` | no |
-| gcs\_bucket\_prefix | Name prefix to be used for GCS Bucket | `string` | `"cmek-encrypted-bucket"` | no |
+| gcs\_bucket\_prefix | Name prefix to be used for GCS Bucket | `string` | `"bkt"` | no |
 | key\_name | Name to be used for KMS Key | `string` | `"crypto-key-example"` | no |
 | key\_rotation\_period | Rotation period in seconds to be used for KMS Key | `string` | `"7776000s"` | no |
 | keyring\_name | Name to be used for KMS Keyring | `string` | `"sample-keyring"` | no |
diff --git a/4-projects/modules/base_env/example_storage_cmek.tf b/4-projects/modules/base_env/example_storage_cmek.tf
@@ -69,7 +69,7 @@ module "gcs_buckets" {
   version            = "~> 3.0"
   project_id         = module.base_shared_vpc_project.project_id
   location           = var.location_gcs
-  name               = "${var.gcs_bucket_prefix}-${lower(var.location_gcs)}-${random_string.bucket_name.result}"
+  name               = "${var.gcs_bucket_prefix}-${module.base_shared_vpc_project.project_id}-${lower(var.location_gcs)}-cmek-encrypted-${random_string.bucket_name.result}"
   bucket_policy_only = true
   encryption = {
     default_kms_key_name = module.kms.keys[var.key_name]
diff --git a/4-projects/modules/base_env/variables.tf b/4-projects/modules/base_env/variables.tf
@@ -107,7 +107,7 @@ variable "key_rotation_period" {
 variable "gcs_bucket_prefix" {
   description = "Name prefix to be used for GCS Bucket"
   type        = string
-  default     = "cmek-encrypted-bucket"
+  default     = "bkt"
 }
 
 variable "remote_state_bucket" {
diff --git a/4-projects/modules/infra_pipelines/README.md b/4-projects/modules/infra_pipelines/README.md
@@ -5,6 +5,7 @@
 |------|-------------|------|---------|:--------:|
 | app\_infra\_repos | A list of Cloud Source Repos to be created to hold app infra Terraform configs. | `list(string)` | n/a | yes |
 | billing\_account | The ID of the billing account to associated this project with. | `string` | n/a | yes |
+| bucket\_prefix | Name prefix to use for state bucket created. | `string` | `"bkt"` | no |
 | cloud\_builder\_artifact\_repo | GAR Repo that stores TF Cloud Builder images. | `string` | n/a | yes |
 | cloudbuild\_project\_id | The project id where the pipelines and repos should be created. | `string` | n/a | yes |
 | default\_region | Default region to create resources where applicable. | `string` | n/a | yes |
diff --git a/4-projects/modules/infra_pipelines/main.tf b/4-projects/modules/infra_pipelines/main.tf
@@ -55,17 +55,24 @@ resource "google_storage_bucket" "cloudbuild_bucket" {
 
 module "tf_workspace" {
   source   = "terraform-google-modules/bootstrap/google//modules/tf_cloudbuild_workspace"
-  version  = "~> 6.3"
+  version  = "~> 6.4"
   for_each = toset(var.app_infra_repos)
 
   project_id = var.cloudbuild_project_id
   location   = var.default_region
 
+  # using bucket custom names for compliance with bucket naming conventions
+  create_state_bucket       = true
+  create_state_bucket_name  = "${var.bucket_prefix}-${var.cloudbuild_project_id}-${each.key}-state"
+  log_bucket_name           = "${var.bucket_prefix}-${var.cloudbuild_project_id}-${each.key}-logs"
+  artifacts_bucket_name     = "${var.bucket_prefix}-${var.cloudbuild_project_id}-${each.key}-artifacts"
   cloudbuild_plan_filename  = "cloudbuild-tf-plan.yaml"
   cloudbuild_apply_filename = "cloudbuild-tf-apply.yaml"
   enable_worker_pool        = true
   worker_pool_id            = var.private_worker_pool_id
   tf_repo_uri               = google_sourcerepo_repository.app_infra_repo[each.key].url
+  create_cloudbuild_sa      = true
+  create_cloudbuild_sa_name = "sa-tf-cb-${each.key}"
   diff_sa_project           = true
   buckets_force_destroy     = true
 
diff --git a/4-projects/modules/infra_pipelines/variables.tf b/4-projects/modules/infra_pipelines/variables.tf
@@ -66,3 +66,9 @@ variable "private_worker_pool_id" {
   description = "ID of the Cloud Build private worker pool."
   type        = string
 }
+
+variable "bucket_prefix" {
+  description = "Name prefix to use for state bucket created."
+  type        = string
+  default     = "bkt"
+}
diff --git a/test/integration/bootstrap/bootstrap_test.go b/test/integration/bootstrap/bootstrap_test.go
@@ -148,7 +148,8 @@ func TestBootstrap(t *testing.T) {
 
 			// cloud build project
 			cbProjectID := bootstrap.GetStringOutput("cloudbuild_project_id")
-			bucketName := terraform.OutputMap(t, bootstrap.GetTFOptions(), "gcs_bucket_cloudbuild_artifacts")
+			artifactsBktName := terraform.OutputMap(t, bootstrap.GetTFOptions(), "gcs_bucket_cloudbuild_artifacts")
+			logsBktName := terraform.OutputMap(t, bootstrap.GetTFOptions(), "gcs_bucket_cloudbuild_logs")
 			defaultRegion := terraform.OutputMap(t, bootstrap.GetTFOptions(), "common_config")["default_region"]
 
 			prj := gcloud.Runf(t, "projects describe %s", cbProjectID)
@@ -170,16 +171,39 @@ func TestBootstrap(t *testing.T) {
 			assert.Equal("VPC_PEERING", globalAddress.Get("purpose").String(), fmt.Sprintf("global address %s purpose should be VPC peering", globalAddressName))
 			assert.Equal(peeredNetworkName, testutils.GetLastSplitElement(globalAddress.Get("network").String(), "/"), fmt.Sprintf("global address %s should be in the peered network", globalAddressName))
 
-			for _, env := range []string{
-				"bootstrap",
-				"org",
-				"env",
-				"net",
-				"proj",
+			for _, bkts := range []struct {
+				env  string
+				repo string
+			}{
+				{
+					env:  "bootstrap",
+					repo: "gcp-bootstrap",
+				},
+				{
+					env:  "org",
+					repo: "gcp-org",
+				},
+				{
+					env:  "env",
+					repo: "gcp-environments",
+				},
+				{
+					env:  "net",
+					repo: "gcp-networks",
+				},
+				{
+					env:  "proj",
+					repo: "gcp-projects",
+				},
 			} {
 				gcAlphaOpts := gcloud.WithCommonArgs([]string{"--project", cbProjectID, "--json"})
-				bkt := gcloud.Run(t, fmt.Sprintf("alpha storage ls --buckets gs://%s", bucketName[env]), gcAlphaOpts).Array()[0]
-				assert.True(bkt.Exists(), "bucket %s should exist", bucketName[env])
+				artifactsBkt := gcloud.Run(t, fmt.Sprintf("alpha storage ls --buckets gs://%s", artifactsBktName[bkts.env]), gcAlphaOpts).Array()[0]
+				assert.True(artifactsBkt.Exists(), "bucket %s should exist", artifactsBktName[bkts.env])
+				assert.Equal(artifactsBktName[bkts.env], fmt.Sprintf("bkt-%s-%s-build-artifacts", cbProjectID, bkts.repo))
+
+				logsBkt := gcloud.Run(t, fmt.Sprintf("alpha storage ls --buckets gs://%s", logsBktName[bkts.env]), gcAlphaOpts).Array()[0]
+				assert.True(logsBkt.Exists(), "bucket %s should exist", logsBktName[bkts.env])
+				assert.Equal(logsBktName[bkts.env], fmt.Sprintf("bkt-%s-%s-build-logs", cbProjectID, bkts.repo))
 			}
 
 			for _, repo := range cloudSourceRepos {
diff --git a/test/integration/projects-shared/projects_shared_test.go b/test/integration/projects-shared/projects_shared_test.go
@@ -22,6 +22,7 @@ import (
 	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
 	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft"
 	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils"
+	"github.com/gruntwork-io/terratest/modules/terraform"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/terraform-google-modules/terraform-example-foundation/test/integration/testutils"
@@ -51,14 +52,17 @@ func TestProjectsShared(t *testing.T) {
 
 	for _, tts := range []struct {
 		name  string
+		repo  string
 		tfDir string
 	}{
 		{
 			name:  "bu1",
+			repo:  "bu1-example-app",
 			tfDir: "../../../4-projects/business_unit_1/shared",
 		},
 		{
 			name:  "bu2",
+			repo:  "bu2-example-app",
 			tfDir: "../../../4-projects/business_unit_2/shared",
 		},
 	} {
@@ -88,6 +92,23 @@ func TestProjectsShared(t *testing.T) {
 					enabledAPIS := gcloud.Runf(t, "services list --project %s", projectID).Array()
 					listApis := testutils.GetResultFieldStrSlice(enabledAPIS, "config.name")
 					assert.Subset(listApis, sharedApisEnabled, "APIs should have been enabled")
+
+					// validate buckets
+					gcAlphaOpts := gcloud.WithCommonArgs([]string{"--project", projectID, "--json"})
+					artifactBktName := terraform.OutputMap(t, shared.GetTFOptions(), "artifact_buckets")[tts.repo]
+					artifactBkt := gcloud.Run(t, fmt.Sprintf("alpha storage ls --buckets gs://%s", artifactBktName), gcAlphaOpts).Array()[0]
+					assert.True(artifactBkt.Exists(), "bucket %s should exist", artifactBktName)
+					assert.Equal(artifactBktName, fmt.Sprintf("bkt-%s-%s-artifacts", projectID, tts.repo))
+
+					logBktName := terraform.OutputMap(t, shared.GetTFOptions(), "log_buckets")[tts.repo]
+					logBkt := gcloud.Run(t, fmt.Sprintf("alpha storage ls --buckets gs://%s", logBktName), gcAlphaOpts).Array()[0]
+					assert.True(logBkt.Exists(), "bucket %s should exist", logBktName)
+					assert.Equal(logBktName, fmt.Sprintf("bkt-%s-%s-logs", projectID, tts.repo))
+
+					stateBktName := terraform.OutputMap(t, shared.GetTFOptions(), "state_buckets")[tts.repo]
+					stateBkt := gcloud.Run(t, fmt.Sprintf("alpha storage ls --buckets gs://%s", stateBktName), gcAlphaOpts).Array()[0]
+					assert.True(stateBkt.Exists(), "bucket %s should exist", stateBktName)
+					assert.Equal(stateBktName, fmt.Sprintf("bkt-%s-%s-state", projectID, tts.repo))
 				})
 			shared.Test()
 		})
