chore: Migrating test Framework for 3 test cases (#1489)

* Migrating Test Framework for 3test cases

* Correcting Build File

* Correcting Build File

* Correcting Build File

* Modifying Files based on Recommendations

* Correcting JSON path

* Modifying the path for networkconfig

* Changing Sanitizer Order to remove Service Account

* Modifying the peering name assert logic

* UnSetting ProjectID to skip ProjectID Sanitiser

* Removing Default Verify in Testing due to issue 1478

Co-authored-by: Akshay Bathija <[email protected]>

Author: akshaybathija-github

.kitchen.yml

@@ -29,15 +29,6 @@ platforms:
   - name: local
 
 suites:
-  - name: "disable_client_cert"
-    driver:
-      root_module_directory: test/fixtures/disable_client_cert
-    verifier:
-      systems:
-        - name: disable_client_cert
-          backend: local
-          controls:
-            - gcloud
   - name: "shared_vpc"
     driver:
       root_module_directory: test/fixtures/shared_vpc
@@ -65,23 +56,6 @@ suites:
       systems:
         - name: simple_regional
           backend: local
-  - name: "private_zonal_with_networking"
-    driver:
-      root_module_directory: test/fixtures/private_zonal_with_networking
-    verifier:
-      systems:
-        - name: private_zonal_with_networking
-          backend: local
-          controls:
-            - gcloud
-        - name: private_zonal_with_networking
-          backend: local
-          controls:
-            - subnet
-        - name: network
-          backend: gcp
-          controls:
-            - network
   - name: "simple_regional_with_networking"
     driver:
       root_module_directory: test/fixtures/simple_regional_with_networking
@@ -188,19 +162,6 @@ suites:
       systems:
         - name: workload_metadata_config
           backend: local
-  - name: "beta_cluster"
-    driver:
-      root_module_directory: test/fixtures/beta_cluster
-    verifier:
-      systems:
-        - name: gcloud
-          backend: local
-          controls:
-            - gcloud
-        - name: gcp
-          backend: gcp
-          controls:
-            - gcp
   - name: "simple_windows_node_pool"
     driver:
       root_module_directory: test/fixtures/simple_windows_node_pool

Makefile

@@ -19,7 +19,7 @@
 SHELL := /usr/bin/env bash
 
 # Pin to 1.3.9 per https://github.com/GoogleCloudPlatform/cloud-foundation-toolkit/issues/1208
-DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.3.9
+DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.8
 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools
 REGISTRY_URL := gcr.io/cloud-foundation-cicd
 DOCKER_BIN ?= docker

build/int.cloudbuild.yaml

@@ -26,21 +26,26 @@ steps:
     - prepare
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create']
-- id: converge disable-client-cert-local
+- id: init disable-client-cert
   waitFor:
-    - create all
+    - prepare
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge disable-client-cert-local']
-- id: verify disable-client-cert-local
+  args: ['/bin/bash', '-c', 'cft test run TestDisableClientCert --stage init --verbose --test-dir test/integration']
+- id: apply disable-client-cert
   waitFor:
-    - converge disable-client-cert-local
+    - init disable-client-cert
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify disable-client-cert-local']
-- id: destroy disable-client-cert-local
+  args: ['/bin/bash', '-c', 'cft test run TestDisableClientCert --stage apply --verbose --test-dir test/integration']
+- id: verify disable-client-cert
   waitFor:
-    - verify disable-client-cert-local
+    - apply disable-client-cert
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy disable-client-cert-local']
+  args: ['/bin/bash', '-c', 'cft test run TestDisableClientCert --stage verify --verbose --test-dir test/integration']
+- id: teardown disable-client-cert
+  waitFor:
+    - verify disable-client-cert
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestDisableClientCert --stage teardown --verbose --test-dir test/integration']
 - id: converge shared-vpc-local
   waitFor:
     - create all
@@ -222,21 +227,26 @@ steps:
     - verify workload-metadata-config-local
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy workload-metadata-config-local']
-- id: converge beta-cluster-local
+- id: init beta-cluster
   waitFor:
-    - create all
+    - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestBetaCluster --stage init --verbose --test-dir test/integration']
+- id: apply beta-cluster
+  waitFor:
+    - init beta-cluster
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge beta-cluster-local']
-- id: verify beta-cluster-local
+  args: ['/bin/bash', '-c', 'cft test run TestBetaCluster --stage apply --verbose --test-dir test/integration']
+- id: verify beta-cluster
   waitFor:
-    - converge beta-cluster-local
+    - apply beta-cluster
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify beta-cluster-local']
-- id: destroy beta-cluster-local
+  args: ['/bin/bash', '-c', 'cft test run TestBetaCluster --stage verify --verbose --test-dir test/integration']
+- id: teardown beta-cluster
   waitFor:
-    - verify beta-cluster-local
+    - verify beta-cluster
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy beta-cluster-local']
+  args: ['/bin/bash', '-c', 'cft test run TestBetaCluster --stage teardown --verbose --test-dir test/integration']
 - id: converge simple-windows-node-pool-local
   waitFor:
     - create all
@@ -372,11 +382,34 @@ steps:
     - verify simple-autopilot-public-local
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-autopilot-public-local']
+- id: init private-zonal-with-networking
+  waitFor:
+    - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestPrivateZonalWithNetworking --stage init --verbose --test-dir test/integration']
+- id: apply private-zonal-with-networking
+  waitFor:
+    - init private-zonal-with-networking
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestPrivateZonalWithNetworking --stage apply --verbose --test-dir test/integration']
+- id: verify private-zonal-with-networking
+  waitFor:
+    - apply private-zonal-with-networking
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestPrivateZonalWithNetworking --stage verify --verbose --test-dir test/integration']
+- id: teardown private-zonal-with-networking
+  waitFor:
+    - verify private-zonal-with-networking
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestPrivateZonalWithNetworking --stage teardown --verbose --test-dir test/integration']
+
+
+
 tags:
 - 'ci'
 - 'integration'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.5'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.8'
 options:
   machineType: 'N1_HIGHCPU_8'

build/lint.cloudbuild.yaml

@@ -22,7 +22,7 @@ tags:
 - 'lint'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.5'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.8'
 options:
   machineType: 'N1_HIGHCPU_8'
   env:

test/fixtures/beta_cluster/network.tf

@@ -27,11 +27,12 @@ resource "google_compute_network" "main" {
 }
 
 resource "google_compute_subnetwork" "main" {
-  name          = "cft-gke-test-${random_string.suffix.result}"
-  ip_cidr_range = "10.0.0.0/17"
-  region        = var.region
-  network       = google_compute_network.main.self_link
-  project       = local.project_id
+  name                     = "cft-gke-test-${random_string.suffix.result}"
+  ip_cidr_range            = "10.0.0.0/17"
+  region                   = var.region
+  network                  = google_compute_network.main.self_link
+  project                  = local.project_id
+  private_ip_google_access = true
 
   secondary_ip_range {
     range_name    = "cft-gke-test-pods-${random_string.suffix.result}"

test/fixtures/beta_cluster/outputs.tf

@@ -64,17 +64,6 @@ output "kubernetes_endpoint" {
   value     = module.this.kubernetes_endpoint
 }
 
-output "client_token" {
-  sensitive = true
-  value     = base64encode(data.google_client_config.default.access_token)
-}
-
-output "ca_certificate" {
-  description = "The cluster CA certificate"
-  value       = module.this.ca_certificate
-  sensitive   = true
-}
-
 output "service_account" {
   description = "The service account to default running nodes as if not overridden in `node_pools`."
   value       = module.this.service_account
@@ -83,7 +72,3 @@ output "service_account" {
 output "database_encryption_key_name" {
   value = google_kms_crypto_key.db.id
 }
-
-output "identity_namespace" {
-  value = module.this.identity_namespace
-}

test/integration/beta_cluster/beta_cluster_test.go

@@ -0,0 +1,91 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package beta_cluster
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
+	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/golden"
+	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft"
+	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBetaCluster(t *testing.T) {
+	gke := tft.NewTFBlueprintTest(t)
+
+	gke.DefineVerify(func(assert *assert.Assertions) {
+		// Commenting Default Verify due to issue 1478 for location Policy
+		// gke.DefaultVerify(assert) //disables no changes
+
+		projectId := gke.GetStringOutput("project_id")
+		location := gke.GetStringOutput("location")
+		clusterName := gke.GetStringOutput("cluster_name")
+		serviceAccount := gke.GetStringOutput("service_account")
+		gcloud.Runf(t, "config set project %s", projectId)
+		op := gcloud.Runf(t, "beta container clusters describe %s --zone %s --project %s", clusterName, location, projectId)
+		// save output as goldenfile
+		g := golden.NewOrUpdate(t, op.String(),
+			golden.WithSanitizer(golden.StringSanitizer(serviceAccount, "SERVICE_ACCOUNT")),
+			golden.WithSanitizer(golden.StringSanitizer(projectId, "PROJECT_ID")),
+			golden.WithSanitizer(golden.StringSanitizer(clusterName, "CLUSTER_NAME")),
+		)
+		// assert json paths against goldenfile data
+		validateJSONPaths := []string{
+			"status",
+			"location",
+			"locations",
+			"privateClusterConfig.enablePrivateEndpoint",
+			"networkConfig.datapathProvider",
+			"podSecurityPolicyConfig.enabled",
+			"databaseEncryption.state",
+			"identityServiceConfig.enabled",
+			"addonsConfig",
+			"networkConfig.datapathProvider",
+			"binaryAuthorization",
+			"podSecurityPolicyConfig",
+			"databaseEncryption.state",
+			"loggingConfig",
+			"monitoringConfig",
+		}
+		for _, pth := range validateJSONPaths {
+			g.JSONEq(assert, op, pth)
+		}
+		for _, np := range op.Get("nodePools").Array() {
+			npName := np.Get("name").String()
+			// sanitze current nodepool data
+			np = g.GetSanitizedJSON(np)
+			// retrive matching nodepool data from goldenfile
+			gNp := utils.GetFirstMatchResult(t, g.GetJSON().Get("nodePools").Array(), "name", npName)
+			switch npName {
+			case "default-pool":
+				assert.False(np.Get("initialNodeCount").Exists(), "has no initial node count")
+				assert.False(np.Get("autoscaling").Exists(), "does not have autoscaling enabled")
+			case "default-node-pool":
+				assert.JSONEq(gNp.Get("config").String(), np.Get("config").String())
+				assert.JSONEq(gNp.Get("autoscaling").String(), np.Get("autoscaling").String())
+				assert.JSONEq(gNp.Get("management").String(), np.Get("management").String())
+			}
+		}
+
+		// verify SA
+		op = gcloud.Runf(t, "iam service-accounts describe %s --project %s", serviceAccount, projectId)
+		assert.Equal(fmt.Sprintf("Terraform-managed service account for cluster %s", clusterName), op.Get("displayName").String(), "has the correct displayname")
+
+	})
+	gke.Test()
+}