Merge branch 'main' into feat/2264/cgroupv2

Author: Ameausoone

README.md

@@ -180,6 +180,7 @@ Then perform the following commands on the root folder:
 | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no |
 | enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no |
 | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no |
+| enterprise\_config | (Optional) Enable or disable GKE enterprise. Valid values are DEFAULT and ENTERPRISE. | `string` | `null` | no |
 | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no |
 | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | <pre>[<br>  "8443",<br>  "9443",<br>  "15017"<br>]</pre> | no |
 | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no |
@@ -356,6 +357,7 @@ The node_pools variable takes the following parameters:
 | placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired |  | Optional |
 | pod_range |  The name of the secondary range for pod IPs. |  | Optional |
 | enable_private_nodes |  Whether nodes have internal IP addresses only. |  | Optional |
+| node_affinity | The node affinty in the format `"{\"key\": \"compute.googleapis.com/node-group-name\", \"operator\": \"IN\", \"values\": [\"node-group-name\"]}"`. |  | Optional |
 | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters |  | Required |
 | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional |
 | node_metadata | Options to expose the node metadata to the workload running on the node | | Optional |

autogen/main/README.md

@@ -245,6 +245,7 @@ The node_pools variable takes the following parameters:
 {% if not private_cluster %}
 | enable_private_nodes |  Whether nodes have internal IP addresses only. |  | Optional |
 {% endif %}
+| node_affinity | The node affinty in the format `"{\"key\": \"compute.googleapis.com/node-group-name\", \"operator\": \"IN\", \"values\": [\"node-group-name\"]}"`. |  | Optional |
 | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters |  | Required |
 | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional |
 | node_metadata | Options to expose the node metadata to the workload running on the node | | Optional |

autogen/main/cluster.tf.tmpl

@@ -260,6 +260,13 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+  dynamic "enterprise_config" {
+    for_each = var.enterprise_config != null ? [1] : []
+      content {
+        desired_tier = var.enterprise_config
+    }
+  }
+
   enable_fqdn_network_policy = var.enable_fqdn_network_policy
 {% if autopilot_cluster %}
   enable_autopilot = true
@@ -571,6 +578,21 @@ resource "google_container_cluster" "primary" {
         }
       }
 
+      dynamic "sole_tenant_config" {
+        # node_affinity is currently the only member of sole_tenant_config
+        for_each = lookup(var.node_pools[0], "node_affinity", null) != null ? [true] : []
+        content {
+          dynamic "node_affinity" {
+            for_each = lookup(var.node_pools[0], "node_affinity", null) != null ? [lookup(var.node_pools[0], "node_affinity", null)] : []
+            content {
+              key = lookup(jsondecode(node_affinity.value), "key", null)
+              operator = lookup(jsondecode(node_affinity.value), "operator", null)
+              values = lookup(jsondecode(node_affinity.value), "values", [])
+            }
+          }
+        }
+      }
+
       service_account = lookup(var.node_pools[0], "service_account", local.service_account)
 
       tags = concat(
@@ -882,7 +904,7 @@ resource "google_container_node_pool" "windows_pools" {
   }
 
   dynamic "network_config" {
-    for_each = length(lookup(each.value, "pod_range", "")) > 0  ? [each.value] : []
+    for_each = length(lookup(each.value, "pod_range", "")) > 0 || {% if private_cluster %}var.enable_private_nodes != null{% else %}lookup(each.value, "enable_private_nodes", null) != null{% endif %} ? [each.value] : []
     content {
       pod_range            = lookup(network_config.value, "pod_range", null)
       {% if private_cluster %}
@@ -1100,6 +1122,21 @@ resource "google_container_node_pool" "windows_pools" {
         pod_pids_limit                         = lookup(each.value, "pod_pids_limit", null)
       }
     }
+
+    dynamic "sole_tenant_config" {
+      # node_affinity is currently the only member of sole_tenant_config
+      for_each = lookup(each.value, "node_affinity", null) != null ? [true] : []
+      content {
+        dynamic "node_affinity" {
+          for_each = lookup(each.value, "node_affinity", null) != null ? [lookup(each.value, "node_affinity", null)] : []
+          content {
+            key = lookup(jsondecode(node_affinity.value), "key", null)
+            operator = lookup(jsondecode(node_affinity.value), "operator", null)
+            values = lookup(jsondecode(node_affinity.value), "values", [])
+          }
+        }
+      }
+    }
     {% if beta_cluster %}
 
     dynamic "sandbox_config" {

autogen/main/variables.tf.tmpl

@@ -1054,3 +1054,9 @@ variable "monitoring_metric_writer_role" {
     error_message = "The monitoring_metric_writer_role must be either a predefined role (roles/*) or a custom role (projects/*/roles/*)."
   }
 }
+
+variable "enterprise_config" {
+  description = "(Optional) Enable or disable GKE enterprise. Valid values are DEFAULT and ENTERPRISE."
+  type        = string
+  default     = null
+}

cluster.tf

@@ -199,6 +199,13 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+  dynamic "enterprise_config" {
+    for_each = var.enterprise_config != null ? [1] : []
+    content {
+      desired_tier = var.enterprise_config
+    }
+  }
+
   enable_fqdn_network_policy = var.enable_fqdn_network_policy
   dynamic "master_authorized_networks_config" {
     for_each = var.gcp_public_cidrs_access_enabled != null || length(var.master_authorized_networks) > 0 ? [true] : []
@@ -441,6 +448,21 @@ resource "google_container_cluster" "primary" {
         }
       }
 
+      dynamic "sole_tenant_config" {
+        # node_affinity is currently the only member of sole_tenant_config
+        for_each = lookup(var.node_pools[0], "node_affinity", null) != null ? [true] : []
+        content {
+          dynamic "node_affinity" {
+            for_each = lookup(var.node_pools[0], "node_affinity", null) != null ? [lookup(var.node_pools[0], "node_affinity", null)] : []
+            content {
+              key      = lookup(jsondecode(node_affinity.value), "key", null)
+              operator = lookup(jsondecode(node_affinity.value), "operator", null)
+              values   = lookup(jsondecode(node_affinity.value), "values", [])
+            }
+          }
+        }
+      }
+
       service_account = lookup(var.node_pools[0], "service_account", local.service_account)
 
       tags = concat(
@@ -595,7 +617,7 @@ resource "google_container_node_pool" "pools" {
   }
 
   dynamic "network_config" {
-    for_each = length(lookup(each.value, "pod_range", "")) > 0 ? [each.value] : []
+    for_each = length(lookup(each.value, "pod_range", "")) > 0 || lookup(each.value, "enable_private_nodes", null) != null ? [each.value] : []
     content {
       pod_range            = lookup(network_config.value, "pod_range", null)
       enable_private_nodes = lookup(network_config.value, "enable_private_nodes", null)
@@ -802,6 +824,21 @@ resource "google_container_node_pool" "pools" {
       }
     }
 
+    dynamic "sole_tenant_config" {
+      # node_affinity is currently the only member of sole_tenant_config
+      for_each = lookup(each.value, "node_affinity", null) != null ? [true] : []
+      content {
+        dynamic "node_affinity" {
+          for_each = lookup(each.value, "node_affinity", null) != null ? [lookup(each.value, "node_affinity", null)] : []
+          content {
+            key      = lookup(jsondecode(node_affinity.value), "key", null)
+            operator = lookup(jsondecode(node_affinity.value), "operator", null)
+            values   = lookup(jsondecode(node_affinity.value), "values", [])
+          }
+        }
+      }
+    }
+
     dynamic "linux_node_config" {
       for_each = length(merge(
         local.node_pools_linux_node_configs_sysctls["all"],
@@ -896,7 +933,7 @@ resource "google_container_node_pool" "windows_pools" {
   }
 
   dynamic "network_config" {
-    for_each = length(lookup(each.value, "pod_range", "")) > 0 ? [each.value] : []
+    for_each = length(lookup(each.value, "pod_range", "")) > 0 || lookup(each.value, "enable_private_nodes", null) != null ? [each.value] : []
     content {
       pod_range            = lookup(network_config.value, "pod_range", null)
       enable_private_nodes = lookup(network_config.value, "enable_private_nodes", null)
@@ -1103,6 +1140,21 @@ resource "google_container_node_pool" "windows_pools" {
       }
     }
 
+    dynamic "sole_tenant_config" {
+      # node_affinity is currently the only member of sole_tenant_config
+      for_each = lookup(each.value, "node_affinity", null) != null ? [true] : []
+      content {
+        dynamic "node_affinity" {
+          for_each = lookup(each.value, "node_affinity", null) != null ? [lookup(each.value, "node_affinity", null)] : []
+          content {
+            key      = lookup(jsondecode(node_affinity.value), "key", null)
+            operator = lookup(jsondecode(node_affinity.value), "operator", null)
+            values   = lookup(jsondecode(node_affinity.value), "values", [])
+          }
+        }
+      }
+    }
+
 
     boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "")
 

examples/node_pool/main.tf

@@ -98,6 +98,12 @@ module "gke" {
       node_count                   = 1
       enable_nested_virtualization = true
     },
+    {
+      name          = "pool-06"
+      node_count    = 1
+      machine_type  = "n1-highmem-96"
+      node_affinity = "{\"key\": \"compute.googleapis.com/node-group-name\", \"operator\": \"IN\", \"values\": [\"${google_compute_node_group.soletenant-nodes.name}\"]}"
+    },
   ]
 
   node_pools_metadata = {
@@ -158,3 +164,18 @@ module "gke" {
     pool-01 = "CGROUP_MODE_V2"
   }
 }
+
+resource "google_compute_node_template" "soletenant-tmpl" {
+  name   = "soletenant-tmpl-${var.cluster_name_suffix}"
+  region = var.region
+
+  node_type = "n1-node-96-624"
+}
+
+resource "google_compute_node_group" "soletenant-nodes" {
+  name = "soletenant-node-group-${var.cluster_name_suffix}"
+  zone = var.zones[0]
+
+  initial_size  = 1
+  node_template = google_compute_node_template.soletenant-tmpl.id
+}

modules/beta-autopilot-private-cluster/README.md

@@ -101,6 +101,7 @@ Then perform the following commands on the root folder:
 | enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no |
 | enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no |
 | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `true` | no |
+| enterprise\_config | (Optional) Enable or disable GKE enterprise. Valid values are DEFAULT and ENTERPRISE. | `string` | `null` | no |
 | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no |
 | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | <pre>[<br>  "8443",<br>  "9443",<br>  "15017"<br>]</pre> | no |
 | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no |

modules/beta-autopilot-private-cluster/cluster.tf

@@ -119,6 +119,13 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+  dynamic "enterprise_config" {
+    for_each = var.enterprise_config != null ? [1] : []
+    content {
+      desired_tier = var.enterprise_config
+    }
+  }
+
   enable_fqdn_network_policy = var.enable_fqdn_network_policy
   enable_autopilot           = true
   dynamic "master_authorized_networks_config" {

modules/beta-autopilot-private-cluster/variables.tf

@@ -622,3 +622,9 @@ variable "monitoring_metric_writer_role" {
     error_message = "The monitoring_metric_writer_role must be either a predefined role (roles/*) or a custom role (projects/*/roles/*)."
   }
 }
+
+variable "enterprise_config" {
+  description = "(Optional) Enable or disable GKE enterprise. Valid values are DEFAULT and ENTERPRISE."
+  type        = string
+  default     = null
+}

modules/beta-autopilot-public-cluster/README.md

@@ -93,6 +93,7 @@ Then perform the following commands on the root folder:
 | enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no |
 | enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no |
 | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `true` | no |
+| enterprise\_config | (Optional) Enable or disable GKE enterprise. Valid values are DEFAULT and ENTERPRISE. | `string` | `null` | no |
 | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no |
 | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | <pre>[<br>  "8443",<br>  "9443",<br>  "15017"<br>]</pre> | no |
 | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no |