feat: add support for NAP/autopilot resource manager tags (#2447)

Author: pib

README.md

@@ -234,7 +234,7 @@ Then perform the following commands on the root folder:
 | network\_policy | Enable network policy addon | `bool` | `false` | no |
 | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no |
 | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no |
-| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no |
+| network\_tags | (Optional) - List of network tags applied to autopilot and auto-provisioned node pools. | `list(string)` | `[]` | no |
 | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no |
 | node\_pools | List of maps containing node pools | `list(map(any))` | <pre>[<br>  {<br>    "name": "default-node-pool"<br>  }<br>]</pre> | no |
 | node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | <pre>{<br>  "all": "",<br>  "default-node-pool": ""<br>}</pre> | no |
@@ -260,6 +260,7 @@ Then perform the following commands on the root folder:
 | registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no |
 | release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no |
+| resource\_manager\_tags | (Optional) - List of resource manager tags applied to autopilot and auto-provisioned node pools. A maximum of 5 tags can be specified. Tags must be in one of these formats: "tagKeys/{tag\_key\_id}"="tagValues/{tag\_value\_id}", "{org\_id}/{tag\_key\_name}"="{tag\_value\_name}", "{project\_id}/{tag\_key\_name}"="{tag\_value\_name}". | `map(string)` | `{}` | no |
 | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no |
 | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no |
 | security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no |

autogen/main/cluster.tf.tmpl

@@ -329,7 +329,7 @@ resource "google_container_cluster" "primary" {
   }
 {% if autopilot_cluster %}
   dynamic "node_pool_auto_config" {
-    for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules || var.insecure_kubelet_readonly_port_enabled != null || var.node_pools_cgroup_mode != null ? [1] : []
+    for_each = length(var.network_tags) > 0 || length(var.resource_manager_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules || var.insecure_kubelet_readonly_port_enabled != null || var.node_pools_cgroup_mode != null ? [1] : []
     content {
       dynamic "network_tags" {
         for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? [1] : []
@@ -338,6 +338,8 @@ resource "google_container_cluster" "primary" {
         }
       }
 
+      resource_manager_tags = length(var.resource_manager_tags) > 0 ? var.resource_manager_tags : null
+
       dynamic "node_kubelet_config" {
         for_each = var.insecure_kubelet_readonly_port_enabled != null ? [1] : []
         content {
@@ -356,7 +358,7 @@ resource "google_container_cluster" "primary" {
 
 {% if autopilot_cluster != true %}
   dynamic "node_pool_auto_config" {
-    for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules || local.node_pools_cgroup_mode != null) ? [1] : []
+    for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || length(var.resource_manager_tags) > 0 || var.add_cluster_firewall_rules || local.node_pools_cgroup_mode != null) ? [1] : []
     content {
       dynamic "network_tags" {
         for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : []
@@ -365,6 +367,8 @@ resource "google_container_cluster" "primary" {
         }
       }
 
+      resource_manager_tags = length(var.resource_manager_tags) > 0 ? var.resource_manager_tags : null
+
       dynamic "linux_node_config" {
         for_each = local.node_pools_cgroup_mode["all"] != "" ? [1] : []
         content {

autogen/main/variables.tf.tmpl

@@ -400,11 +400,17 @@ variable "node_pools_oauth_scopes" {
 {% endif %}
 
 variable "network_tags" {
-  description = "(Optional) - List of network tags applied to auto-provisioned node pools."
+  description = "(Optional) - List of network tags applied to autopilot and auto-provisioned node pools."
   type = list(string)
   default = []
 }
 
+variable "resource_manager_tags" {
+  description = "(Optional) - List of resource manager tags applied to autopilot and auto-provisioned node pools. A maximum of 5 tags can be specified. Tags must be in one of these formats: \"tagKeys/{tag_key_id}\"=\"tagValues/{tag_value_id}\", \"{org_id}/{tag_key_name}\"=\"{tag_value_name}\", \"{project_id}/{tag_key_name}\"=\"{tag_value_name}\"."
+  type = map(string)
+  default = {}
+}
+
 variable "enable_k8s_beta_apis" {
   description = "(Optional) - List of Kubernetes Beta APIs to enable in cluster."
   type        = list(string)

cluster.tf

@@ -265,7 +265,7 @@ resource "google_container_cluster" "primary" {
   }
 
   dynamic "node_pool_auto_config" {
-    for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules || local.node_pools_cgroup_mode != null) ? [1] : []
+    for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || length(var.resource_manager_tags) > 0 || var.add_cluster_firewall_rules || local.node_pools_cgroup_mode != null) ? [1] : []
     content {
       dynamic "network_tags" {
         for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : []
@@ -274,6 +274,8 @@ resource "google_container_cluster" "primary" {
         }
       }
 
+      resource_manager_tags = length(var.resource_manager_tags) > 0 ? var.resource_manager_tags : null
+
       dynamic "linux_node_config" {
         for_each = local.node_pools_cgroup_mode["all"] != "" ? [1] : []
         content {

examples/node_pool/main.tf

@@ -26,6 +26,20 @@ provider "kubernetes" {
   cluster_ca_certificate = base64decode(module.gke.ca_certificate)
 }
 
+resource "google_tags_tag_key" "key" {
+  parent     = "projects/${var.project_id}"
+  short_name = "key${var.cluster_name_suffix}"
+  purpose    = "GCE_FIREWALL"
+  purpose_data = {
+    network = "${var.project_id}/${var.network}"
+  }
+}
+
+resource "google_tags_tag_value" "value" {
+  parent     = google_tags_tag_key.key.id
+  short_name = "value${var.cluster_name_suffix}"
+}
+
 module "gke" {
   source  = "terraform-google-modules/kubernetes-engine/google//modules/beta-public-cluster"
   version = "~> 39.0"
@@ -47,6 +61,10 @@ module "gke" {
   logging_variant                   = "MAX_THROUGHPUT"
   dns_allow_external_traffic        = true
 
+  resource_manager_tags = {
+    "${var.project_id}/${google_tags_tag_key.key.short_name}" = google_tags_tag_value.value.short_name
+  }
+
   node_pools = [
     {
       name            = "pool-01"

metadata.display.yaml

@@ -390,6 +390,9 @@ spec:
         remove_default_node_pool:
           name: remove_default_node_pool
           title: Remove Default Node Pool
+        resource_manager_tags:
+          name: resource_manager_tags
+          title: Resource Manager Tags
         resource_usage_export_dataset_id:
           name: resource_usage_export_dataset_id
           title: Resource Usage Export Dataset Id

metadata.yaml

@@ -404,9 +404,13 @@ spec:
             - https://www.googleapis.com/auth/cloud-platform
           default-node-pool: []
       - name: network_tags
-        description: (Optional) - List of network tags applied to auto-provisioned node pools.
+        description: (Optional) - List of network tags applied to autopilot and auto-provisioned node pools.
         varType: list(string)
         defaultValue: []
+      - name: resource_manager_tags
+        description: "(Optional) - List of resource manager tags applied to autopilot and auto-provisioned node pools. A maximum of 5 tags can be specified. Tags must be in one of these formats: \"tagKeys/{tag_key_id}\"=\"tagValues/{tag_value_id}\", \"{org_id}/{tag_key_name}\"=\"{tag_value_name}\", \"{project_id}/{tag_key_name}\"=\"{tag_value_name}\"."
+        varType: map(string)
+        defaultValue: {}
       - name: enable_k8s_beta_apis
         description: (Optional) - List of Kubernetes Beta APIs to enable in cluster.
         varType: list(string)

modules/beta-autopilot-private-cluster/README.md

@@ -144,7 +144,7 @@ Then perform the following commands on the root folder:
 | name | The name of the cluster (required) | `string` | n/a | yes |
 | network | The VPC network to host the cluster in (required) | `string` | n/a | yes |
 | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no |
-| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no |
+| network\_tags | (Optional) - List of network tags applied to autopilot and auto-provisioned node pools. | `list(string)` | `[]` | no |
 | node\_pools\_cgroup\_mode | Specifies the Linux cgroup mode for autopilot Kubernetes nodes in the cluster. Accepted values are `CGROUP_MODE_UNSPECIFIED`, `CGROUP_MODE_V1`, and `CGROUP_MODE_V2`, which determine the control group hierarchy used for resource management. | `string` | `null` | no |
 | notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no |
 | notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no |
@@ -156,6 +156,7 @@ Then perform the following commands on the root folder:
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no |
 | registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no |
 | release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no |
+| resource\_manager\_tags | (Optional) - List of resource manager tags applied to autopilot and auto-provisioned node pools. A maximum of 5 tags can be specified. Tags must be in one of these formats: "tagKeys/{tag\_key\_id}"="tagValues/{tag\_value\_id}", "{org\_id}/{tag\_key\_name}"="{tag\_value\_name}", "{project\_id}/{tag\_key\_name}"="{tag\_value\_name}". | `map(string)` | `{}` | no |
 | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no |
 | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no |
 | security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no |

modules/beta-autopilot-private-cluster/cluster.tf

@@ -185,7 +185,7 @@ resource "google_container_cluster" "primary" {
     }
   }
   dynamic "node_pool_auto_config" {
-    for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules || var.insecure_kubelet_readonly_port_enabled != null || var.node_pools_cgroup_mode != null ? [1] : []
+    for_each = length(var.network_tags) > 0 || length(var.resource_manager_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules || var.insecure_kubelet_readonly_port_enabled != null || var.node_pools_cgroup_mode != null ? [1] : []
     content {
       dynamic "network_tags" {
         for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? [1] : []
@@ -194,6 +194,8 @@ resource "google_container_cluster" "primary" {
         }
       }
 
+      resource_manager_tags = length(var.resource_manager_tags) > 0 ? var.resource_manager_tags : null
+
       dynamic "node_kubelet_config" {
         for_each = var.insecure_kubelet_readonly_port_enabled != null ? [1] : []
         content {

modules/beta-autopilot-private-cluster/metadata.display.yaml

@@ -292,6 +292,9 @@ spec:
         release_channel:
           name: release_channel
           title: Release Channel
+        resource_manager_tags:
+          name: resource_manager_tags
+          title: Resource Manager Tags
         resource_usage_export_dataset_id:
           name: resource_usage_export_dataset_id
           title: Resource Usage Export Dataset Id