Add Resource Manager Tags support to 'google_container_cluster' (#9531) (#7001)

* resourceManagerTags added to Cluster Node Config schema

* update beta tag

* add cluster and node proto tests

* add expand and flatten proto

* removed beta tag

* added to documentation

* added resource manager tags to auto pilot

* migrating resourceManagerTags tests

* migrating node_pools test

* migrating additional tests

* minor fixes

* fixing tests

* add in-place update support

* fixed tests

* fixed annotations

* validated clusters and node pools tests. Isolated node pool auto config

* isolated resource manager tags from docs

* fixed permission issue

* fixed spaces

* fixed non determinism on tag keys

* removed auto_pilot rmts

* fixed time_sleep

* add depends_on to IAM policies

[upstream:343ff46df5008cc457392c3baafd0acc6dc97633]

Signed-off-by: Modular Magician <[email protected]>

Author: modular-magician

.changelog/9531.txt

@@ -0,0 +1,6 @@
+```release-note:enhancement
+container: added `node_config.resource_manager_tags` field to `google_container_cluster` resource
+```
+```release-note:enhancement
+container: added `node_config.resource_manager_tags` field to `google_container_node_pool` resource
+```

google-beta/services/compute/resource_compute_instance.go

@@ -660,7 +660,7 @@ func ResourceComputeInstance() *schema.Resource {
 				Optional: true,
 				Elem:     &schema.Schema{Type: schema.TypeString},
 				Description: `A set of key/value label pairs assigned to the instance.
-				
+
 				**Note**: This field is non-authoritative, and will only manage the labels present in your configuration.
 				Please refer to the field 'effective_labels' for all of the labels present on the resource.`,
 			},

google-beta/services/container/node_config.go

@@ -664,6 +664,11 @@ func schemaNodeConfig() *schema.Schema {
 						},
 					},
 				},
+				"resource_manager_tags": {
+					Type:        schema.TypeMap,
+					Optional:    true,
+					Description: `A map of resource manager tags. Resource manager tag keys and values have the same definition as resource manager tags. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456. The field is ignored (both PUT & PATCH) when empty.`,
+				},
 				"enable_confidential_storage": {
 					Type:        schema.TypeBool,
 					Optional:    true,
@@ -867,6 +872,10 @@ func expandNodeConfig(v interface{}) *container.NodeConfig {
 		nc.ResourceLabels = m
 	}
 
+	if v, ok := nodeConfig["resource_manager_tags"]; ok && len(v.(map[string]interface{})) > 0 {
+		nc.ResourceManagerTags = expandResourceManagerTags(v)
+	}
+
 	if v, ok := nodeConfig["tags"]; ok {
 		tagsList := v.([]interface{})
 		tags := []string{}
@@ -965,6 +974,19 @@ func expandNodeConfig(v interface{}) *container.NodeConfig {
 	return nc
 }
 
+func expandResourceManagerTags(v interface{}) *container.ResourceManagerTags {
+	rmts := make(map[string]string)
+
+	if v != nil {
+		rmts = tpgresource.ConvertStringMap(v.(map[string]interface{}))
+	}
+
+	return &container.ResourceManagerTags{
+		Tags:            rmts,
+		ForceSendFields: []string{"Tags"},
+	}
+}
+
 func expandWorkloadMetadataConfig(v interface{}) *container.WorkloadMetadataConfig {
 	if v == nil {
 		return nil
@@ -1182,8 +1204,8 @@ func flattenNodeConfig(c *container.NodeConfig, v interface{}) []map[string]inte
 		"advanced_machine_features":          flattenAdvancedMachineFeaturesConfig(c.AdvancedMachineFeatures),
 		"sole_tenant_config":                 flattenSoleTenantConfig(c.SoleTenantConfig),
 		"fast_socket":                        flattenFastSocket(c.FastSocket),
-
-		"enable_confidential_storage": c.EnableConfidentialStorage,
+		"resource_manager_tags":              flattenResourceManagerTags(c.ResourceManagerTags),
+		"enable_confidential_storage":        c.EnableConfidentialStorage,
 	})
 
 	if len(c.OauthScopes) > 0 {
@@ -1193,6 +1215,19 @@ func flattenNodeConfig(c *container.NodeConfig, v interface{}) []map[string]inte
 	return config
 }
 
+func flattenResourceManagerTags(c *container.ResourceManagerTags) map[string]interface{} {
+	rmt := make(map[string]interface{})
+
+	if c != nil {
+		for k, v := range c.Tags {
+			rmt[k] = v
+		}
+
+	}
+
+	return rmt
+}
+
 func flattenAdvancedMachineFeaturesConfig(c *container.AdvancedMachineFeatures) []map[string]interface{} {
 	result := []map[string]interface{}{}
 	if c != nil {

google-beta/services/container/resource_container_cluster.go

@@ -92,6 +92,7 @@ var (
 	forceNewClusterNodeConfigFields = []string{
 		"labels",
 		"workload_metadata_config",
+		"resource_manager_tags",
 	}
 
 	suppressDiffForAutopilot = schema.SchemaDiffSuppressFunc(func(k, oldValue, newValue string, d *schema.ResourceData) bool {
@@ -5349,6 +5350,7 @@ func expandNodePoolAutoConfig(configured interface{}) *container.NodePoolAutoCon
 	if v, ok := config["network_tags"]; ok && len(v.([]interface{})) > 0 {
 		npac.NetworkTags = expandNodePoolAutoConfigNetworkTags(v)
 	}
+
 	return npac
 }
 

google-beta/services/container/resource_container_cluster_test.go

@@ -57,6 +57,43 @@ func TestAccContainerCluster_basic(t *testing.T) {
 	})
 }
 
+func TestAccContainerCluster_resourceManagerTags(t *testing.T) {
+	t.Parallel()
+
+	pid := envvar.GetTestProjectFromEnv()
+
+	randomSuffix := acctest.RandString(t, 10)
+	clusterName := fmt.Sprintf("tf-test-cluster-%s", randomSuffix)
+
+	networkName := acctest.BootstrapSharedTestNetwork(t, "gke-cluster")
+	subnetworkName := acctest.BootstrapSubnet(t, "gke-cluster", networkName)
+
+	acctest.VcrTest(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+		ExternalProviders: map[string]resource.ExternalProvider{
+			"time": {},
+		},
+		CheckDestroy: testAccCheckContainerClusterDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccContainerCluster_resourceManagerTags(pid, clusterName, networkName, subnetworkName, randomSuffix),
+				Check: resource.ComposeTestCheckFunc(
+					resource.TestCheckResourceAttrSet("google_container_cluster.primary", "self_link"),
+					resource.TestCheckResourceAttrSet("google_container_cluster.primary", "node_config.0.resource_manager_tags.%"),
+				),
+			},
+			{
+				ResourceName:            "google_container_cluster.primary",
+				ImportStateId:           fmt.Sprintf("us-central1-a/%s", clusterName),
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"min_master_version", "deletion_protection"},
+			},
+		},
+	})
+}
+
 func TestAccContainerCluster_networkingModeRoutes(t *testing.T) {
 	t.Parallel()
 
@@ -4418,11 +4455,11 @@ func testAccContainerCluster_withIncompatibleMasterVersionNodeVersion(name strin
 	resource "google_container_cluster" "gke_cluster" {
 		name = "%s"
 		location = "us-central1"
-	
+
 		min_master_version = "1.10.9-gke.5"
 		node_version = "1.10.6-gke.11"
 		initial_node_count = 1
-		
+
 	}
 	`, name)
 }
@@ -6631,7 +6668,7 @@ resource "google_container_cluster" "with_autoprovisioning" {
   min_master_version = data.google_container_engine_versions.central1a.latest_master_version
   initial_node_count = 1
   deletion_protection = false
-  
+
   network    = "%s"
   subnetwork    = "%s"
 
@@ -9197,7 +9234,7 @@ resource "google_compute_resource_policy" "policy" {
 resource "google_container_cluster" "cluster" {
   name               = "%s"
   location           = "us-central1-a"
-  
+
   node_pool {
     name               = "%s"
     initial_node_count = 2
@@ -9504,3 +9541,86 @@ func testAccContainerCluster_withWorkloadALTSConfig(projectID, name, networkName
 	}
 `, projectID, networkName, subnetworkName, name, enable)
 }
+
+func testAccContainerCluster_resourceManagerTags(projectID, clusterName, networkName, subnetworkName, randomSuffix string) string {
+	return fmt.Sprintf(`
+data "google_project" "project" {
+  project_id = "%[1]s"
+}
+
+resource "google_project_iam_binding" "tagHoldAdmin" {
+  project = "%[1]s"
+  role    = "roles/resourcemanager.tagHoldAdmin"
+  members = [
+    "serviceAccount:service-${data.google_project.project.number}@container-engine-robot.iam.gserviceaccount.com",
+  ]
+}
+
+resource "google_project_iam_binding" "tagUser" {
+  project = "%[1]s"
+  role    = "roles/resourcemanager.tagUser"
+  members = [
+    "serviceAccount:service-${data.google_project.project.number}@container-engine-robot.iam.gserviceaccount.com",
+    "serviceAccount:${data.google_project.project.number}@cloudservices.gserviceaccount.com",
+  ]
+
+  depends_on = [google_project_iam_binding.tagHoldAdmin]
+}
+
+resource "time_sleep" "wait_120_seconds" {
+  create_duration = "120s"
+
+  depends_on = [
+    google_project_iam_binding.tagHoldAdmin,
+    google_project_iam_binding.tagUser
+  ]
+}
+
+resource "google_tags_tag_key" "key" {
+  parent = "projects/%[1]s"
+  short_name = "foobarbaz-%[2]s"
+  description = "For foo/bar resources"
+  purpose = "GCE_FIREWALL"
+  purpose_data = {
+    network = "%[1]s/%[4]s"
+  }
+}
+
+resource "google_tags_tag_value" "value" {
+  parent = "tagKeys/${google_tags_tag_key.key.name}"
+  short_name = "foo-%[2]s"
+  description = "For foo resources"
+}
+
+data "google_container_engine_versions" "uscentral1a" {
+  location = "us-central1-a"
+}
+
+resource "google_container_cluster" "primary" {
+  name               = "%[3]s"
+  location           = "us-central1-a"
+  min_master_version = data.google_container_engine_versions.uscentral1a.release_channel_latest_version["STABLE"]
+  initial_node_count = 1
+
+  node_config {
+    machine_type    = "n1-standard-1"  // can't be e2 because of local-ssd
+    disk_size_gb    = 15
+
+    resource_manager_tags = {
+      "tagKeys/${google_tags_tag_key.key.name}" = "tagValues/${google_tags_tag_value.value.name}"
+    }
+  }
+
+  deletion_protection = false
+  network    = "%[4]s"
+  subnetwork    = "%[5]s"
+
+  timeouts {
+    create = "30m"
+    update = "40m"
+  }
+
+  depends_on = [time_sleep.wait_120_seconds]
+}
+`, projectID, randomSuffix, clusterName, networkName, subnetworkName)
+}

google-beta/services/container/resource_container_node_pool.go

@@ -1586,6 +1586,48 @@ func nodePoolUpdate(d *schema.ResourceData, meta interface{}, nodePoolInfo *Node
 			log.Printf("[INFO] Updated tags for node pool %s", name)
 		}
 
+		if d.HasChange(prefix + "node_config.0.resource_manager_tags") {
+			req := &container.UpdateNodePoolRequest{
+				Name: name,
+			}
+			if v, ok := d.GetOk(prefix + "node_config.0.resource_manager_tags"); ok {
+				req.ResourceManagerTags = expandResourceManagerTags(v)
+			}
+
+			// sets resource manager tags to the empty list when user removes a previously defined list of tags entriely
+			// aka the node pool goes from having tags to no longer having any
+			if req.ResourceManagerTags == nil {
+				tags := make(map[string]string)
+				rmTags := &container.ResourceManagerTags{
+					Tags: tags,
+				}
+				req.ResourceManagerTags = rmTags
+			}
+
+			updateF := func() error {
+				clusterNodePoolsUpdateCall := config.NewContainerClient(userAgent).Projects.Locations.Clusters.NodePools.Update(nodePoolInfo.fullyQualifiedName(name), req)
+				if config.UserProjectOverride {
+					clusterNodePoolsUpdateCall.Header().Add("X-Goog-User-Project", nodePoolInfo.project)
+				}
+				op, err := clusterNodePoolsUpdateCall.Do()
+				if err != nil {
+					return err
+				}
+
+				// Wait until it's updated
+				return ContainerOperationWait(config, op,
+					nodePoolInfo.project,
+					nodePoolInfo.location,
+					"updating GKE node pool resource manager tags", userAgent,
+					timeout)
+			}
+
+			if err := retryWhileIncompatibleOperation(timeout, npLockKey, updateF); err != nil {
+				return err
+			}
+			log.Printf("[INFO] Updated resource manager tags for node pool %s", name)
+		}
+
 		if d.HasChange(prefix + "node_config.0.resource_labels") {
 			req := &container.UpdateNodePoolRequest{
 				Name: name,