Merge branch 'master' into filestore_autopilot

Author: apeabody

README.md

@@ -190,6 +190,7 @@ Then perform the following commands on the root folder:
 | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no |
 | identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no |
 | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no |
+| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no |
 | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes |
@@ -319,6 +320,7 @@ The node_pools variable takes the following parameters:
 | gpu_partition_size | Size of partitions to create on the GPU | null | Optional |
 | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional |
 | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional |
+| insecure_kubelet_readonly_port_enabled | (boolean) Whether or not to enable the insecure Kubelet readonly port. | null | Optional |
 | key | The key required for the taint | | Required |
 | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT.  | DEFAULT | Optional |
 | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume.  | 0 | Optional |

autogen/main/README.md

@@ -213,6 +213,7 @@ The node_pools variable takes the following parameters:
 | gpu_partition_size | Size of partitions to create on the GPU | null | Optional |
 | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional |
 | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional |
+| insecure_kubelet_readonly_port_enabled | (boolean) Whether or not to enable the insecure Kubelet readonly port. | null | Optional |
 | key | The key required for the taint | | Required |
 | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT.  | DEFAULT | Optional |
 | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume.  | 0 | Optional |

autogen/main/cluster.tf.tmpl

@@ -530,6 +530,21 @@ resource "google_container_cluster" "primary" {
         }
       }
 
+      dynamic "kubelet_config" {
+        for_each = length(setintersection(
+          keys(var.node_pools[0]),
+          ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"]
+        )) != 0 || var.insecure_kubelet_readonly_port_enabled != null ? [1] : []
+
+        content {
+          cpu_manager_policy                     = lookup(var.node_pools[0], "cpu_manager_policy", "static")
+          cpu_cfs_quota                          = lookup(var.node_pools[0], "cpu_cfs_quota", null)
+          cpu_cfs_quota_period                   = lookup(var.node_pools[0], "cpu_cfs_quota_period", null)
+          insecure_kubelet_readonly_port_enabled = lookup(var.node_pools[0], "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled) != null ? upper(tostring(lookup(var.node_pools[0], "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled))) : null
+          pod_pids_limit                         = lookup(var.node_pools[0], "pod_pids_limit", null)
+        }
+      }
+
       service_account = lookup(var.node_pools[0], "service_account", local.service_account)
 
       tags = concat(
@@ -661,7 +676,6 @@ resource "google_container_cluster" "primary" {
       }
     }
   }
-  {% if beta_cluster %}
 
   node_pool_defaults {
     node_config_defaults {
@@ -675,15 +689,17 @@ resource "google_container_cluster" "primary" {
       }
       {% endif %}
       {% if autopilot_cluster != true %}
+      {% if beta_cluster %}
       gcfs_config {
         enabled = var.enable_gcfs
       }
       {% endif %}
+      insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled != null ? upper(tostring(var.insecure_kubelet_readonly_port_enabled)) : null
+      {% endif %}
     }
   }
-  {% endif %}
-  {% if beta_cluster %}
 
+  {% if beta_cluster %}
   depends_on = [google_project_iam_member.service_agent]
   {% endif %}
 }
@@ -706,6 +722,9 @@ locals {
     "enable_secure_boot",
     "enable_integrity_monitoring",
     "local_ssd_count",
+    {% if beta_cluster %}
+    "local_ssd_ephemeral_count",
+    {% endif %}
     "machine_type",
     "placement_policy",
     "max_pods_per_node",
@@ -723,6 +742,7 @@ locals {
     "reservation_affinity_key",
     "reservation_affinity_values",
     "enable_confidential_nodes",
+    "secondary_boot_disk",
   ]
 }
 
@@ -1042,14 +1062,15 @@ resource "google_container_node_pool" "windows_pools" {
     dynamic "kubelet_config" {
       for_each = length(setintersection(
         keys(each.value),
-        ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"]
+        ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"]
       )) != 0 ? [1] : []
 
       content {
-        cpu_manager_policy   = lookup(each.value, "cpu_manager_policy", "static")
-        cpu_cfs_quota        = lookup(each.value, "cpu_cfs_quota", null)
-        cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
-        pod_pids_limit       = lookup(each.value, "pod_pids_limit", null)
+        cpu_manager_policy                     = lookup(each.value, "cpu_manager_policy", "static")
+        cpu_cfs_quota                          = lookup(each.value, "cpu_cfs_quota", null)
+        cpu_cfs_quota_period                   = lookup(each.value, "cpu_cfs_quota_period", null)
+        insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null) != null ? upper(tostring(each.value.insecure_kubelet_readonly_port_enabled)) : null
+        pod_pids_limit                         = lookup(each.value, "pod_pids_limit", null)
       }
     }
     {% if beta_cluster %}
@@ -1088,7 +1109,7 @@ resource "google_container_node_pool" "windows_pools" {
     }
 
     dynamic "confidential_nodes" {
-      for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.confidential_nodes] : []
+      for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : []
       content {
         enabled = confidential_nodes.value
       }

autogen/main/dns.tf.tmpl

@@ -20,7 +20,7 @@
   Manage kube-dns configmaps
  *****************************************/
 
-resource "kubernetes_config_map_v1_data" "kube-dns" {
+resource "kubernetes_config_map_v1_data" "kube_dns" {
   count = local.custom_kube_dns_config && !local.upstream_nameservers_config ? 1 : 0
 
   metadata {
@@ -44,7 +44,7 @@ EOF
   ]
 }
 
-resource "kubernetes_config_map_v1_data" "kube-dns-upstream-namservers" {
+resource "kubernetes_config_map_v1_data" "kube_dns_upstream_nameservers" {
   count = !local.custom_kube_dns_config && local.upstream_nameservers_config ? 1 : 0
 
   metadata {
@@ -68,7 +68,7 @@ EOF
   ]
 }
 
-resource "kubernetes_config_map_v1_data" "kube-dns-upstream-nameservers-and-stub-domains" {
+resource "kubernetes_config_map_v1_data" "kube_dns_upstream_nameservers_and_stub_domains" {
   count = local.custom_kube_dns_config && local.upstream_nameservers_config ? 1 : 0
 
   metadata {

autogen/main/masq.tf.tmpl

@@ -19,7 +19,7 @@
 /******************************************
   Create ip-masq-agent confimap
  *****************************************/
-resource "kubernetes_config_map" "ip-masq-agent" {
+resource "kubernetes_config_map" "ip_masq_agent" {
   count = var.configure_ip_masq ? 1 : 0
 
   metadata {

autogen/main/moved.tf.tmpl

@@ -0,0 +1,62 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# Updates for kebab to snake case, to match best practices and Google style.
+moved {
+  from = kubernetes_config_map_v1_data.kube-dns
+  to   = kubernetes_config_map_v1_data.kube_dns
+}
+
+# Typo fix and snake case at the same time
+moved {
+  from = kubernetes_config_map_v1_data.kube-dns-upstream-namservers
+  to   = kubernetes_config_map_v1_data.kube_dns_upstream_nameservers
+}
+
+moved {
+  from = kubernetes_config_map_v1_data.kube-dns-upstream-nameservers-and-stub-domains
+  to   = kubernetes_config_map_v1_data.kube_dns_upstream_nameservers_and_stub_domains
+}
+
+moved {
+  from = kubernetes_config_map.ip-masq-agent
+  to   = kubernetes_config_map.ip_masq_agent
+}
+
+moved {
+  from = google_project_iam_member.cluster_service_account-nodeService_account
+  to   = google_project_iam_member.cluster_service_account_node_service_account
+}
+
+moved {
+  from = google_project_iam_member.cluster_service_account-metric_writer
+  to   = google_project_iam_member.cluster_service_account_metric_writer
+}
+
+moved {
+  from = google_project_iam_member.cluster_service_account-resourceMetadata-writer
+  to   = google_project_iam_member.cluster_service_account_resource_metadata_writer
+}
+
+moved {
+  from = google_project_iam_member.cluster_service_account-gcr
+  to   = google_project_iam_member.cluster_service_account_gcr
+}
+
+moved {
+  from = google_project_iam_member.cluster_service_account-artifact-registry
+  to   = google_project_iam_member.cluster_service_account_artifact_registry
+}

autogen/main/sa.tf.tmpl

@@ -46,35 +46,35 @@ resource "google_service_account" "cluster_service_account" {
   display_name = "Terraform-managed service account for cluster ${var.name}"
 }
 
-resource "google_project_iam_member" "cluster_service_account-nodeService_account" {
+resource "google_project_iam_member" "cluster_service_account_node_service_account" {
   count   = var.create_service_account ? 1 : 0
   project = google_service_account.cluster_service_account[0].project
   role    = "roles/container.defaultNodeServiceAccount"
   member  = google_service_account.cluster_service_account[0].member
 }
 
-resource "google_project_iam_member" "cluster_service_account-metric_writer" {
+resource "google_project_iam_member" "cluster_service_account_metric_writer" {
   count   = var.create_service_account ? 1 : 0
   project = google_service_account.cluster_service_account[0].project
   role    = "roles/monitoring.metricWriter"
   member  = google_service_account.cluster_service_account[0].member
 }
 
-resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" {
+resource "google_project_iam_member" "cluster_service_account_resource_metadata_writer" {
   count   = var.create_service_account ? 1 : 0
   project = google_service_account.cluster_service_account[0].project
   role    = "roles/stackdriver.resourceMetadata.writer"
   member  = google_service_account.cluster_service_account[0].member
 }
 
-resource "google_project_iam_member" "cluster_service_account-gcr" {
+resource "google_project_iam_member" "cluster_service_account_gcr" {
   for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : []
   project  = each.key
   role     = "roles/storage.objectViewer"
   member   = "serviceAccount:${google_service_account.cluster_service_account[0].email}"
 }
 
-resource "google_project_iam_member" "cluster_service_account-artifact-registry" {
+resource "google_project_iam_member" "cluster_service_account_artifact_registry" {
   for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : []
   project  = each.key
   role     = "roles/artifactregistry.reader"

autogen/main/variables.tf.tmpl

@@ -109,6 +109,12 @@ variable "service_external_ips" {
 }
 
 {% if autopilot_cluster != true %}
+variable "insecure_kubelet_readonly_port_enabled" {
+  type = bool
+  description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`."
+  default     = null
+}
+
 variable "datapath_provider" {
   type        = string
   description = "The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature."

autogen/main/versions.tf.tmpl

@@ -24,11 +24,11 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = ">= 5.40.0, < 7"
+      version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7"
     }
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 5.40.0, < 7"
+      version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"
@@ -86,7 +86,7 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = ">= 5.40.0, < 7"
+      version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"

autogen/safer-cluster/main.tf.tmpl

@@ -119,11 +119,16 @@ module "gke" {
   //   All applications should run with an identity defined via Workload Identity anyway.
   // - Use a service account passed as a parameter to the module, in case the user
   //   wants to maintain control of their service accounts.
-  create_service_account = var.compute_engine_service_account == "" ? true : false
   service_account        = var.compute_engine_service_account
   registry_project_ids   = var.registry_project_ids
   grant_registry_access  = var.grant_registry_access
 
+  // If create_service_account is explicitly set to false we short-circuit the
+  // compute_engine_service_account check to potentially avoid an error (see variables.tf documentation).
+  // Otherwise if true (the default), we check if compute_engine_service_account is set for backwards compatability
+  // before the create_service_account variable was added.
+  create_service_account = var.create_service_account == false ? var.create_service_account : (var.compute_engine_service_account == "" ? true : false)
+
   issue_client_certificate = false
 
   cluster_resource_labels = var.cluster_resource_labels
@@ -209,7 +214,7 @@ module "gke" {
   // Enabling vulnerability and audit for workloads
   workload_vulnerability_mode = var.workload_vulnerability_mode
   workload_config_audit_mode  = var.workload_config_audit_mode
-  
+
   // Enabling security posture
   security_posture_mode               = var.security_posture_mode
   security_posture_vulnerability_mode = var.security_posture_vulnerability_mode