@@ -15,15 +15,19 @@ package node_pool
1515
1616import (
1717 "fmt"
18+ "slices"
19+ "strings"
1820 "testing"
1921 "time"
2022
23+ "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/cai"
2124 "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
25+ "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/golden"
2226 "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft"
27+ "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils"
2328 "github.com/gruntwork-io/terratest/modules/k8s"
2429 "github.com/stretchr/testify/assert"
2530 "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils"
26- gkeutils "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/utils"
2731)
2832
2933func TestNodePool (t * testing.T ) {
@@ -34,20 +38,25 @@ func TestNodePool(t *testing.T) {
3438 bpt .DefineVerify (func (assert * assert.Assertions ) {
3539 // Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token
3640 // bpt.DefaultVerify(assert)
37- gkeutils .TGKEVerify (t , bpt , assert ) // Verify Resources
41+ testutils .TGKEVerify (t , bpt , assert ) // Verify Resources
3842
3943 projectId := bpt .GetStringOutput ("project_id" )
4044 location := bpt .GetStringOutput ("location" )
4145 clusterName := bpt .GetStringOutput ("cluster_name" )
46+ randomString := bpt .GetStringOutput ("random_string" )
47+ kubernetesEndpoint := bpt .GetStringOutput ("kubernetes_endpoint" )
48+ //serviceAccount := bpt.GetStringOutput("service_account")
4249
43- //cluster := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId)
50+ // Retrieve Cluster using CAI
4451 clusterResourceName := fmt .Sprintf ("//container.googleapis.com/projects/%s/locations/%s/clusters/%s" , projectId , location , clusterName )
45- cluster := gkeutils .GetProjectResources (t , projectId , gkeutils .WithAssetType ("container.googleapis.com/Cluster" )).Get ("#(name=\" " + clusterResourceName + "\" ).resource.data" )
52+ cluster := cai .GetProjectResources (t , projectId , cai .WithAssetTypes ([]string {"container.googleapis.com/Cluster" })).Get ("#(name=\" " + clusterResourceName + "\" ).resource.data" )
53+ // Equivalent gcloud describe command
54+ // cluster := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId)
4655
4756 // Cluster
4857 assert .Contains ([]string {"RUNNING" , "RECONCILING" }, cluster .Get ("status" ).String (), "Cluster is Running" )
4958 assert .Equal ("COS_CONTAINERD" , cluster .Get ("autoscaling.autoprovisioningNodePoolDefaults.imageType" ).String (), "has the expected image type" )
50- assert .Equal ("[ \n \" https://www.googleapis.com/auth/cloud-platform\" \n ]" , cluster .Get ("autoscaling.autoprovisioningNodePoolDefaults.oauthScopes" ).String (), "has the expected oauth scopes" )
59+ assert .Equal ("https://www.googleapis.com/auth/cloud-platform" , cluster .Get ("autoscaling.autoprovisioningNodePoolDefaults.oauthScopes.0 " ).String (), "has the expected oauth scopes" )
5160 assert .Equal ("default" , cluster .Get ("autoscaling.autoprovisioningNodePoolDefaults.serviceAccount" ).String (), "has the expected service account" )
5261 assert .Equal ("OPTIMIZE_UTILIZATION" , cluster .Get ("autoscaling.autoscalingProfile" ).String (), "has the expected autoscaling profile" )
5362 assert .True (cluster .Get ("autoscaling.enableNodeAutoprovisioning" ).Bool (), "has the expected node autoprovisioning" )
@@ -65,6 +74,70 @@ func TestNodePool(t *testing.T) {
6574 ]` ,
6675 cluster .Get ("autoscaling.resourceLimits" ).String (), "has the expected resource limits" )
6776
77+ // Cluster (using golden image with sanitizer)
78+ g := golden .NewOrUpdate (t , cluster .String (),
79+ //golden.WithSanitizer(golden.StringSanitizer(serviceAccount, "SERVICE_ACCOUNT")),
80+ golden .WithSanitizer (golden .StringSanitizer (projectId , "PROJECT_ID" )),
81+ golden .WithSanitizer (golden .StringSanitizer (location , "LOCATION" )),
82+ //golden.WithSanitizer(golden.StringSanitizer(clusterName, "CLUSTER_NAME")),
83+ golden .WithSanitizer (golden .StringSanitizer (randomString , "RANDOM_STRING" )),
84+ golden .WithSanitizer (golden .StringSanitizer (kubernetesEndpoint , "KUBERNETES_ENDPOINT" )),
85+ )
86+
87+ fmt .Println ("START JSONEq" )
88+ validateJSONPaths := []string {
89+ "autoscaling.autoprovisioningNodePoolDefaults.imageType" ,
90+ "autoscaling.autoprovisioningNodePoolDefaults.oauthScopes.0" ,
91+ "autoscaling.autoprovisioningNodePoolDefaults.serviceAccount" ,
92+ "autoscaling.autoscalingProfile" ,
93+ "autoscaling.enableNodeAutoprovisioning" ,
94+ "autoscaling.resourceLimits[0].maximum" ,
95+ "autoscaling.resourceLimits[0].minimum" ,
96+ "autoscaling.resourceLimits[0].resourceType" ,
97+ "autoscaling.resourceLimits[1].maximum" ,
98+ "autoscaling.resourceLimits[1].minimum" ,
99+ "autoscaling.resourceLimits[1].resourceType" ,
100+ }
101+ for _ , pth := range validateJSONPaths {
102+ g .JSONEq (assert , cluster , pth )
103+ }
104+ fmt .Println ("END JSONEq" )
105+
106+ fmt .Println ("START one path" )
107+ g .JSONPathEqs (assert , cluster , []string {"autoscaling.autoprovisioningNodePoolDefaults.imageType" })
108+ fmt .Println ("END one path" )
109+
110+ fmt .Println ("START multi path" )
111+ g .JSONPathEqs (assert , cluster , validateJSONPaths )
112+ fmt .Println ("END multi path" )
113+
114+ fmt .Println ("START all paths" )
115+ // Test validating all paths in golden image
116+ jsonPaths := utils .GetTerminalJSONPaths (g .GetJSON ())
117+
118+ // List of paths exempt from validation
119+ exemptJSONPathPrefixes := []string {
120+ "nodePools" , // nodePools are unordered
121+ "monitoringConfig.componentConfig.enableComponents" ,
122+ }
123+
124+ // Remove exempt paths by prefix
125+ jsonPaths = slices .DeleteFunc (jsonPaths , func (s string ) bool {
126+ for _ , path := range exemptJSONPathPrefixes {
127+ if strings .HasPrefix (s , path ) {
128+ // prefix match
129+ return true
130+ }
131+ }
132+ // no prefix match
133+ return false
134+ })
135+
136+ jsonPaths = append (jsonPaths , "monitoringConfig.componentConfig.enableComponents" )
137+ g .JSONPathEqs (assert , cluster , jsonPaths )
138+
139+ fmt .Println ("END all paths" )
140+
68141 // Pool-01
69142 assert .Equal ("pool-01" , cluster .Get ("nodePools.#(name==\" pool-01\" ).name" ).String (), "pool-1 exists" )
70143 assert .Equal ("e2-medium" , cluster .Get ("nodePools.#(name==\" pool-01\" ).config.machineType" ).String (), "is the expected machine type" )
@@ -129,7 +202,7 @@ func TestNodePool(t *testing.T) {
129202 k8sOpts := k8s.KubectlOptions {}
130203 clusterNodesOp , err := k8s .RunKubectlAndGetOutputE (t , & k8sOpts , "get" , "nodes" , "-o" , "json" )
131204 assert .NoError (err )
132- clusterNodes := testutils .ParseKubectlJSONResult (t , clusterNodesOp )
205+ clusterNodes := utils .ParseKubectlJSONResult (t , clusterNodesOp )
133206 assert .JSONEq (`[
134207 {
135208 "effect": "PreferNoSchedule",
@@ -148,6 +221,11 @@ func TestNodePool(t *testing.T) {
148221 "effect": "PreferNoSchedule",
149222 "key": "all-pools-example",
150223 "value": "true"
224+ },
225+ {
226+ "effect": "NoSchedule",
227+ "key": "nvidia.com/gpu",
228+ "value": "present"
151229 }
152230 ]` ,
153231 clusterNodes .Get ("items.#(metadata.labels.node_pool==\" pool-02\" ).spec.taints" ).String (), "has the expected all-pools-example taint" )
@@ -156,6 +234,11 @@ func TestNodePool(t *testing.T) {
156234 "effect": "PreferNoSchedule",
157235 "key": "all-pools-example",
158236 "value": "true"
237+ },
238+ {
239+ "effect": "NoSchedule",
240+ "key": "sandbox.gke.io/runtime",
241+ "value": "gvisor"
159242 }
160243 ]` ,
161244 clusterNodes .Get ("items.#(metadata.labels.node_pool==\" pool-03\" ).spec.taints" ).String (), "has the expected all-pools-example taint" )
0 commit comments