feat(autopilot): add insecure_kubelet_readonly_port_enabled (#2252)

apeabody · web-flow · commit 47a49ac3471a · 2025-01-24T10:24:03.000-08:00
diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl
@@ -279,10 +279,17 @@ resource "google_container_cluster" "primary" {
   }
 {% if autopilot_cluster %}
   dynamic "node_pool_auto_config" {
-    for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? [1] : []
+    for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules || var.insecure_kubelet_readonly_port_enabled != null ? [1] : []
     content {
       network_tags {
-        tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags
+        tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : length(var.network_tags) > 0 ? var.network_tags : null
+      }
+
+      dynamic "node_kubelet_config" {
+        for_each = var.insecure_kubelet_readonly_port_enabled != null ? [1] : []
+        content {
+          insecure_kubelet_readonly_port_enabled = upper(tostring(var.insecure_kubelet_readonly_port_enabled))
+        }
       }
     }
   }
diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl
@@ -108,13 +108,13 @@ variable "service_external_ips" {
   default     = false
 }
 
-{% if autopilot_cluster != true %}
 variable "insecure_kubelet_readonly_port_enabled" {
   type = bool
-  description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`."
+  description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters.{% if autopilot_cluster != true %} Note: this can be set at the node pool level separately within `node_pools`.{% endif %}"
   default     = null
 }
 
+{% if autopilot_cluster != true %}
 variable "datapath_provider" {
   type        = string
   description = "The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature."
diff --git a/examples/simple_autopilot_private/main.tf b/examples/simple_autopilot_private/main.tf
@@ -37,18 +37,19 @@ module "gke" {
   source  = "terraform-google-modules/kubernetes-engine/google//modules/beta-autopilot-private-cluster"
   version = "~> 35.0"
 
-  project_id                      = var.project_id
-  name                            = "${local.cluster_type}-cluster"
-  regional                        = true
-  region                          = var.region
-  network                         = module.gcp-network.network_name
-  subnetwork                      = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)]
-  ip_range_pods                   = local.pods_range_name
-  ip_range_services               = local.svc_range_name
-  release_channel                 = "REGULAR"
-  enable_vertical_pod_autoscaling = true
-  enable_private_endpoint         = true
-  enable_private_nodes            = true
-  network_tags                    = [local.cluster_type]
-  deletion_protection             = false
+  project_id                             = var.project_id
+  name                                   = "${local.cluster_type}-cluster"
+  regional                               = true
+  region                                 = var.region
+  network                                = module.gcp-network.network_name
+  subnetwork                             = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)]
+  ip_range_pods                          = local.pods_range_name
+  ip_range_services                      = local.svc_range_name
+  release_channel                        = "REGULAR"
+  enable_vertical_pod_autoscaling        = true
+  enable_private_endpoint                = true
+  enable_private_nodes                   = true
+  network_tags                           = [local.cluster_type]
+  deletion_protection                    = false
+  insecure_kubelet_readonly_port_enabled = false
 }
diff --git a/modules/beta-autopilot-private-cluster/README.md b/modules/beta-autopilot-private-cluster/README.md
@@ -114,6 +114,7 @@ Then perform the following commands on the root folder:
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no |
 | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no |
 | identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no |
+| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. | `bool` | `null` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no |
 | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes |
diff --git a/modules/beta-autopilot-private-cluster/cluster.tf b/modules/beta-autopilot-private-cluster/cluster.tf
@@ -135,10 +135,17 @@ resource "google_container_cluster" "primary" {
     }
   }
   dynamic "node_pool_auto_config" {
-    for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? [1] : []
+    for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules || var.insecure_kubelet_readonly_port_enabled != null ? [1] : []
     content {
       network_tags {
-        tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags
+        tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : length(var.network_tags) > 0 ? var.network_tags : null
+      }
+
+      dynamic "node_kubelet_config" {
+        for_each = var.insecure_kubelet_readonly_port_enabled != null ? [1] : []
+        content {
+          insecure_kubelet_readonly_port_enabled = upper(tostring(var.insecure_kubelet_readonly_port_enabled))
+        }
       }
     }
   }
diff --git a/modules/beta-autopilot-private-cluster/variables.tf b/modules/beta-autopilot-private-cluster/variables.tf
@@ -108,6 +108,12 @@ variable "service_external_ips" {
   default     = false
 }
 
+variable "insecure_kubelet_readonly_port_enabled" {
+  type        = bool
+  description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters."
+  default     = null
+}
+
 variable "maintenance_start_time" {
   type        = string
   description = "Time window specified for daily or recurring maintenance operations in RFC3339 format"
diff --git a/modules/beta-autopilot-public-cluster/README.md b/modules/beta-autopilot-public-cluster/README.md
@@ -106,6 +106,7 @@ Then perform the following commands on the root folder:
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no |
 | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no |
 | identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no |
+| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. | `bool` | `null` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no |
 | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes |
diff --git a/modules/beta-autopilot-public-cluster/cluster.tf b/modules/beta-autopilot-public-cluster/cluster.tf
@@ -135,10 +135,17 @@ resource "google_container_cluster" "primary" {
     }
   }
   dynamic "node_pool_auto_config" {
-    for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? [1] : []
+    for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules || var.insecure_kubelet_readonly_port_enabled != null ? [1] : []
     content {
       network_tags {
-        tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags
+        tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : length(var.network_tags) > 0 ? var.network_tags : null
+      }
+
+      dynamic "node_kubelet_config" {
+        for_each = var.insecure_kubelet_readonly_port_enabled != null ? [1] : []
+        content {
+          insecure_kubelet_readonly_port_enabled = upper(tostring(var.insecure_kubelet_readonly_port_enabled))
+        }
       }
     }
   }
diff --git a/modules/beta-autopilot-public-cluster/variables.tf b/modules/beta-autopilot-public-cluster/variables.tf
@@ -108,6 +108,12 @@ variable "service_external_ips" {
   default     = false
 }
 
+variable "insecure_kubelet_readonly_port_enabled" {
+  type        = bool
+  description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters."
+  default     = null
+}
+
 variable "maintenance_start_time" {
   type        = string
   description = "Time window specified for daily or recurring maintenance operations in RFC3339 format"
diff --git a/terraform-google-kubernetes-engine b/terraform-google-kubernetes-engine
diff --git a/test/integration/simple_regional_with_gateway_api/testdata/TestSimpleRegionalWithGatewayAPI.json b/test/integration/simple_regional_with_gateway_api/testdata/TestSimpleRegionalWithGatewayAPI.json
@@ -213,7 +213,7 @@
         "podRange": "cft-gke-test-pods-RANDOM_STRING"
       },
       "podIpv4CidrSize": 24,
-      "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/clusters/gke-simple-regional-gatewayapi-cluster-RANDOM_STRING/nodePools/default-pool",
+      "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/clusters/simple-regional-gatewayapi-cluster-RANDOM_STRING/nodePools/default-pool",
       "status": "RUNNING",
       "upgradeSettings": {
         "maxSurge": 1,
diff --git a/test/integration/testutils/utils.go b/test/integration/testutils/utils.go
@@ -1,4 +1,4 @@
-// Copyright 2022-2024 Google LLC
+// Copyright 2022-2025 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -27,7 +27,6 @@ import (
 	tfjson "github.com/hashicorp/terraform-json"
 	"github.com/stretchr/testify/assert"
 	"github.com/tidwall/gjson"
-	"golang.org/x/sync/errgroup"
 )
 
 var (
@@ -40,6 +39,9 @@ var (
 
 		// API Rate limit exceeded errors can be retried.
 		".*rateLimitExceeded.*": "Rate limit exceeded.",
+
+		// Internal errors can be retried
+		".*Error code 13, message: an internal error has occurred": "Internal error.",
 	}
 
 	ClusterAlwaysExemptPaths = []string{"nodePools"} // node pools are separately checked by name
@@ -100,16 +102,10 @@ func TGKEAssertGolden(assert *assert.Assertions, golden *golden.GoldenFile, clus
 
 		nodeCheckPaths := utils.GetTerminalJSONPaths(golden.GetJSON().Get(fmt.Sprintf("nodePools.#(name==%s)", nodePool)))
 
-		syncGroup := new(errgroup.Group)
-		syncGroup.SetLimit(24)
 		for _, nodeCheckPath := range nodeCheckPaths {
-			nodeCheckPath := nodeCheckPath
-			syncGroup.Go(func() error {
-				gotData := golden.ApplySanitizers(clusterJson.Get(fmt.Sprintf("nodePools.#(name==%s)", nodePool)).Get(nodeCheckPath).String())
-				gfData := golden.GetJSON().Get(fmt.Sprintf("nodePools.#(name==%s)", nodePool)).Get(nodeCheckPath).String()
-				assert.Equalf(gfData, gotData, "For node %s path %q expected %q to match fixture %q", nodePool, nodeCheckPath, gotData, gfData)
-				return nil
-			})
+			gotData := golden.ApplySanitizers(clusterJson.Get(fmt.Sprintf("nodePools.#(name==%s)", nodePool)).Get(nodeCheckPath).String())
+			gfData := golden.GetJSON().Get(fmt.Sprintf("nodePools.#(name==%s)", nodePool)).Get(nodeCheckPath).String()
+			assert.Equalf(gfData, gotData, "For node %q path %q expected %q to match fixture %q", nodePool, nodeCheckPath, gotData, gfData)
 		}
 	}
 }

Original file line number	Diff line number	Diff line change
`@@ -279,10 +279,17 @@ resource "google_container_cluster" "primary" {`
`279`	`279`	`}`
`280`	`280`	`{% if autopilot_cluster %}`
`281`	`281`	`dynamic "node_pool_auto_config" {`
`282`		`- for_each = length(var.network_tags) > 0 \|\| var.add_cluster_firewall_rules \|\| var.add_master_webhook_firewall_rules \|\| var.add_shadow_firewall_rules ? [1] : []`
	`282`	`+ for_each = length(var.network_tags) > 0 \|\| var.add_cluster_firewall_rules \|\| var.add_master_webhook_firewall_rules \|\| var.add_shadow_firewall_rules \|\| var.insecure_kubelet_readonly_port_enabled != null ? [1] : []`
`283`	`283`	`content {`
`284`	`284`	`network_tags {`
`285`		`- tags = var.add_cluster_firewall_rules \|\| var.add_master_webhook_firewall_rules \|\| var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags`
	`285`	`+ tags = var.add_cluster_firewall_rules \|\| var.add_master_webhook_firewall_rules \|\| var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : length(var.network_tags) > 0 ? var.network_tags : null`
	`286`	`+ }`
	`287`	`+`
	`288`	`+ dynamic "node_kubelet_config" {`
	`289`	`+ for_each = var.insecure_kubelet_readonly_port_enabled != null ? [1] : []`
	`290`	`+ content {`
	`291`	`+ insecure_kubelet_readonly_port_enabled = upper(tostring(var.insecure_kubelet_readonly_port_enabled))`
	`292`	`+ }`
`286`	`293`	`}`
`287`	`294`	`}`
`288`	`295`	`}`
Original file line number	Diff line number	Diff line change
`@@ -108,13 +108,13 @@ variable "service_external_ips" {`
`108`	`108`	`default = false`
`109`	`109`	`}`
`110`	`110`
`111`		`-{% if autopilot_cluster != true %}`
`112`	`111`	`variable "insecure_kubelet_readonly_port_enabled" {`
`113`	`112`	`type = bool`
`114`		- description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`."
	`113`	+ description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters.{% if autopilot_cluster != true %} Note: this can be set at the node pool level separately within `node_pools`.{% endif %}"
`115`	`114`	`default = null`
`116`	`115`	`}`
`117`	`116`
	`117`	`+{% if autopilot_cluster != true %}`
`118`	`118`	`variable "datapath_provider" {`
`119`	`119`	`type = string`
`120`	`120`	description = "The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature."
Original file line number	Diff line number	Diff line change
`@@ -135,10 +135,17 @@ resource "google_container_cluster" "primary" {`
`135`	`135`	`}`
`136`	`136`	`}`
`137`	`137`	`dynamic "node_pool_auto_config" {`
`138`		`- for_each = length(var.network_tags) > 0 \|\| var.add_cluster_firewall_rules \|\| var.add_master_webhook_firewall_rules \|\| var.add_shadow_firewall_rules ? [1] : []`
	`138`	`+ for_each = length(var.network_tags) > 0 \|\| var.add_cluster_firewall_rules \|\| var.add_master_webhook_firewall_rules \|\| var.add_shadow_firewall_rules \|\| var.insecure_kubelet_readonly_port_enabled != null ? [1] : []`
`139`	`139`	`content {`
`140`	`140`	`network_tags {`
`141`		`- tags = var.add_cluster_firewall_rules \|\| var.add_master_webhook_firewall_rules \|\| var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags`
	`141`	`+ tags = var.add_cluster_firewall_rules \|\| var.add_master_webhook_firewall_rules \|\| var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : length(var.network_tags) > 0 ? var.network_tags : null`
	`142`	`+ }`
	`143`	`+`
	`144`	`+ dynamic "node_kubelet_config" {`
	`145`	`+ for_each = var.insecure_kubelet_readonly_port_enabled != null ? [1] : []`
	`146`	`+ content {`
	`147`	`+ insecure_kubelet_readonly_port_enabled = upper(tostring(var.insecure_kubelet_readonly_port_enabled))`
	`148`	`+ }`
`142`	`149`	`}`
`143`	`150`	`}`
`144`	`151`	`}`