feat: Add support for gVisor per node pool (#1001)

LukaszCzarnotaSabre · bharathkkb · web-flow · commit 850c4181ec93 · 2021-09-27T20:20:44.000-04:00
* feat: Add support for gVisor per node pool

* fix image type, add test

Co-authored-by: Bharath KKB &lt;bharathkrishnakb@gmail.com&gt;
diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl
@@ -495,7 +495,7 @@ resource "google_container_node_pool" "pools" {
 
   node_config {
     {% if beta_cluster %}
-    image_type   = lookup(each.value, "image_type", var.sandbox_enabled ? "COS_CONTAINERD" : "COS")
+    image_type   = lookup(each.value, "image_type", lookup(each.value, "sandbox_enabled", var.sandbox_enabled) ? "COS_CONTAINERD" : "COS")
     {% else %}
     image_type   = lookup(each.value, "image_type", "COS")
     {% endif %}
@@ -577,8 +577,7 @@ resource "google_container_node_pool" "pools" {
     }
     {% if beta_cluster %}
     dynamic "sandbox_config" {
-      for_each = local.cluster_sandbox_enabled
-
+      for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : []
       content {
         sandbox_type = sandbox_config.value
       }
diff --git a/autogen/main/main.tf.tmpl b/autogen/main/main.tf.tmpl
@@ -105,8 +105,6 @@ locals {
 
   cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }]
 
-  cluster_sandbox_enabled = var.sandbox_enabled ? ["gvisor"] : []
-
 {% endif %}
 
   cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{
diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf
@@ -70,14 +70,15 @@ module "gke" {
     },
     {
       name            = "pool-03"
+      machine_type    = "n1-standard-2"
       node_locations  = "${var.region}-b,${var.region}-c"
       autoscaling     = false
       node_count      = 2
       disk_type       = "pd-standard"
-      image_type      = "COS"
       auto_upgrade    = true
       service_account = var.compute_engine_service_account
       pod_range       = "test"
+      sandbox_enabled = true
     },
   ]
 
diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf
@@ -447,7 +447,7 @@ resource "google_container_node_pool" "pools" {
   }
 
   node_config {
-    image_type   = lookup(each.value, "image_type", var.sandbox_enabled ? "COS_CONTAINERD" : "COS")
+    image_type   = lookup(each.value, "image_type", lookup(each.value, "sandbox_enabled", var.sandbox_enabled) ? "COS_CONTAINERD" : "COS")
     machine_type = lookup(each.value, "machine_type", "e2-medium")
     labels = merge(
       lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
@@ -523,8 +523,7 @@ resource "google_container_node_pool" "pools" {
       }
     }
     dynamic "sandbox_config" {
-      for_each = local.cluster_sandbox_enabled
-
+      for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : []
       content {
         sandbox_type = sandbox_config.value
       }
diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf
@@ -96,8 +96,6 @@ locals {
 
   cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }]
 
-  cluster_sandbox_enabled = var.sandbox_enabled ? ["gvisor"] : []
-
 
   cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{
     security_group = var.authenticator_security_group
diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf
@@ -363,7 +363,7 @@ resource "google_container_node_pool" "pools" {
   }
 
   node_config {
-    image_type   = lookup(each.value, "image_type", var.sandbox_enabled ? "COS_CONTAINERD" : "COS")
+    image_type   = lookup(each.value, "image_type", lookup(each.value, "sandbox_enabled", var.sandbox_enabled) ? "COS_CONTAINERD" : "COS")
     machine_type = lookup(each.value, "machine_type", "e2-medium")
     labels = merge(
       lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
@@ -439,8 +439,7 @@ resource "google_container_node_pool" "pools" {
       }
     }
     dynamic "sandbox_config" {
-      for_each = local.cluster_sandbox_enabled
-
+      for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : []
       content {
         sandbox_type = sandbox_config.value
       }
diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf
@@ -96,8 +96,6 @@ locals {
 
   cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }]
 
-  cluster_sandbox_enabled = var.sandbox_enabled ? ["gvisor"] : []
-
 
   cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{
     security_group = var.authenticator_security_group
diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf
@@ -428,7 +428,7 @@ resource "google_container_node_pool" "pools" {
   }
 
   node_config {
-    image_type   = lookup(each.value, "image_type", var.sandbox_enabled ? "COS_CONTAINERD" : "COS")
+    image_type   = lookup(each.value, "image_type", lookup(each.value, "sandbox_enabled", var.sandbox_enabled) ? "COS_CONTAINERD" : "COS")
     machine_type = lookup(each.value, "machine_type", "e2-medium")
     labels = merge(
       lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
@@ -504,8 +504,7 @@ resource "google_container_node_pool" "pools" {
       }
     }
     dynamic "sandbox_config" {
-      for_each = local.cluster_sandbox_enabled
-
+      for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : []
       content {
         sandbox_type = sandbox_config.value
       }
diff --git a/modules/beta-public-cluster-update-variant/main.tf b/modules/beta-public-cluster-update-variant/main.tf
@@ -96,8 +96,6 @@ locals {
 
   cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }]
 
-  cluster_sandbox_enabled = var.sandbox_enabled ? ["gvisor"] : []
-
 
   cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{
     security_group = var.authenticator_security_group
diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf
@@ -344,7 +344,7 @@ resource "google_container_node_pool" "pools" {
   }
 
   node_config {
-    image_type   = lookup(each.value, "image_type", var.sandbox_enabled ? "COS_CONTAINERD" : "COS")
+    image_type   = lookup(each.value, "image_type", lookup(each.value, "sandbox_enabled", var.sandbox_enabled) ? "COS_CONTAINERD" : "COS")
     machine_type = lookup(each.value, "machine_type", "e2-medium")
     labels = merge(
       lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
@@ -420,8 +420,7 @@ resource "google_container_node_pool" "pools" {
       }
     }
     dynamic "sandbox_config" {
-      for_each = local.cluster_sandbox_enabled
-
+      for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : []
       content {
         sandbox_type = sandbox_config.value
       }
diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf
@@ -96,8 +96,6 @@ locals {
 
   cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }]
 
-  cluster_sandbox_enabled = var.sandbox_enabled ? ["gvisor"] : []
-
 
   cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{
     security_group = var.authenticator_security_group
diff --git a/test/integration/node_pool/controls/gcloud.rb b/test/integration/node_pool/controls/gcloud.rb
@@ -351,7 +351,7 @@
             including(
               "name" => "pool-03",
               "config" => including(
-                "machineType" => "e2-medium",
+                "machineType" => "n1-standard-2",
               ),
             )
           )
@@ -408,6 +408,7 @@
                   "all-pools-example" => "true",
                   "cluster_name" => cluster_name,
                   "node_pool" => "pool-03",
+                  "sandbox.gke.io/runtime"=>"gvisor"
                 },
               ),
             )
@@ -441,6 +442,17 @@
           )
         end
 
+        it "has the expected image" do
+          expect(data['nodePools']).to include(
+            including(
+              "name" => "pool-03",
+              "config" => including(
+                "imageType" => "COS_CONTAINERD",
+              ),
+            )
+          )
+        end
+
         it "has the expected linux node config sysctls" do
           expect(data['nodePools']).to include(
             including(

Original file line number	Diff line number	Diff line change
`@@ -447,7 +447,7 @@ resource "google_container_node_pool" "pools" {`
`447`	`447`	`}`
`448`	`448`
`449`	`449`	`node_config {`
`450`		`- image_type = lookup(each.value, "image_type", var.sandbox_enabled ? "COS_CONTAINERD" : "COS")`
	`450`	`+ image_type = lookup(each.value, "image_type", lookup(each.value, "sandbox_enabled", var.sandbox_enabled) ? "COS_CONTAINERD" : "COS")`
`451`	`451`	`machine_type = lookup(each.value, "machine_type", "e2-medium")`
`452`	`452`	`labels = merge(`
`453`	`453`	`lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},`
`@@ -523,8 +523,7 @@ resource "google_container_node_pool" "pools" {`
`523`	`523`	`}`
`524`	`524`	`}`
`525`	`525`	`dynamic "sandbox_config" {`
`526`		`- for_each = local.cluster_sandbox_enabled`
`527`		`-`
	`526`	`+ for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : []`
`528`	`527`	`content {`
`529`	`528`	`sandbox_type = sandbox_config.value`
`530`	`529`	`}`
Original file line number	Diff line number	Diff line change
`@@ -363,7 +363,7 @@ resource "google_container_node_pool" "pools" {`
`363`	`363`	`}`
`364`	`364`
`365`	`365`	`node_config {`
`366`		`- image_type = lookup(each.value, "image_type", var.sandbox_enabled ? "COS_CONTAINERD" : "COS")`
	`366`	`+ image_type = lookup(each.value, "image_type", lookup(each.value, "sandbox_enabled", var.sandbox_enabled) ? "COS_CONTAINERD" : "COS")`
`367`	`367`	`machine_type = lookup(each.value, "machine_type", "e2-medium")`
`368`	`368`	`labels = merge(`
`369`	`369`	`lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},`
`@@ -439,8 +439,7 @@ resource "google_container_node_pool" "pools" {`
`439`	`439`	`}`
`440`	`440`	`}`
`441`	`441`	`dynamic "sandbox_config" {`
`442`		`- for_each = local.cluster_sandbox_enabled`
`443`		`-`
	`442`	`+ for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : []`
`444`	`443`	`content {`
`445`	`444`	`sandbox_type = sandbox_config.value`
`446`	`445`	`}`
Original file line number	Diff line number	Diff line change
`@@ -428,7 +428,7 @@ resource "google_container_node_pool" "pools" {`
`428`	`428`	`}`
`429`	`429`
`430`	`430`	`node_config {`
`431`		`- image_type = lookup(each.value, "image_type", var.sandbox_enabled ? "COS_CONTAINERD" : "COS")`
	`431`	`+ image_type = lookup(each.value, "image_type", lookup(each.value, "sandbox_enabled", var.sandbox_enabled) ? "COS_CONTAINERD" : "COS")`
`432`	`432`	`machine_type = lookup(each.value, "machine_type", "e2-medium")`
`433`	`433`	`labels = merge(`
`434`	`434`	`lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},`
`@@ -504,8 +504,7 @@ resource "google_container_node_pool" "pools" {`
`504`	`504`	`}`
`505`	`505`	`}`
`506`	`506`	`dynamic "sandbox_config" {`
`507`		`- for_each = local.cluster_sandbox_enabled`
`508`		`-`
	`507`	`+ for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : []`
`509`	`508`	`content {`
`510`	`509`	`sandbox_type = sandbox_config.value`
`511`	`510`	`}`
Original file line number	Diff line number	Diff line change
`@@ -344,7 +344,7 @@ resource "google_container_node_pool" "pools" {`
`344`	`344`	`}`
`345`	`345`
`346`	`346`	`node_config {`
`347`		`- image_type = lookup(each.value, "image_type", var.sandbox_enabled ? "COS_CONTAINERD" : "COS")`
	`347`	`+ image_type = lookup(each.value, "image_type", lookup(each.value, "sandbox_enabled", var.sandbox_enabled) ? "COS_CONTAINERD" : "COS")`
`348`	`348`	`machine_type = lookup(each.value, "machine_type", "e2-medium")`
`349`	`349`	`labels = merge(`
`350`	`350`	`lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},`
`@@ -420,8 +420,7 @@ resource "google_container_node_pool" "pools" {`
`420`	`420`	`}`
`421`	`421`	`}`
`422`	`422`	`dynamic "sandbox_config" {`
`423`		`- for_each = local.cluster_sandbox_enabled`
`424`		`-`
	`423`	`+ for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : []`
`425`	`424`	`content {`
`426`	`425`	`sandbox_type = sandbox_config.value`
`427`	`426`	`}`