Support additional_ip_ranges_config (adding multiple subnets to a cluster) (#14521) (#10451)

[upstream:d84e82b1e7cb4fd67fc922a8c0826731040b00f2]

Signed-off-by: Modular Magician <[email protected]>

Author: modular-magician

.changelog/14521.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+container: Support additional_ip_ranges_config (adding multiple subnets to a cluster).
+```

google-beta/services/container/resource_container_cluster.go

@@ -1822,6 +1822,27 @@ func ResourceContainerCluster() *schema.Resource {
 								},
 							},
 						},
+						"additional_ip_ranges_config": {
+							Type:        schema.TypeList,
+							Optional:    true,
+							Description: `AdditionalIPRangesConfig is the configuration for individual additional subnetworks attached to the cluster`,
+							Elem: &schema.Resource{
+								Schema: map[string]*schema.Schema{
+									"subnetwork": {
+										Type:             schema.TypeString,
+										Required:         true,
+										DiffSuppressFunc: tpgresource.CompareSelfLinkOrResourceName,
+										Description:      `Name of the subnetwork. This can be the full path of the subnetwork or just the name.`,
+									},
+									"pod_ipv4_range_names": {
+										Type:        schema.TypeList,
+										Optional:    true,
+										Description: `List of secondary ranges names within this subnetwork that can be used for pod IPs.`,
+										Elem:        &schema.Schema{Type: schema.TypeString},
+									},
+								},
+							},
+						},
 					},
 				},
 			},
@@ -2663,7 +2684,7 @@ func resourceContainerClusterCreate(d *schema.ResourceData, meta interface{}) er
 		}
 	}
 
-	ipAllocationBlock, err := expandIPAllocationPolicy(d.Get("ip_allocation_policy"), d.Get("networking_mode").(string), d.Get("enable_autopilot").(bool))
+	ipAllocationBlock, aircs, err := expandIPAllocationPolicy(d.Get("ip_allocation_policy"), d, d.Get("networking_mode").(string), d.Get("enable_autopilot").(bool), config)
 	if err != nil {
 		return err
 	}
@@ -2889,6 +2910,10 @@ func resourceContainerClusterCreate(d *schema.ResourceData, meta interface{}) er
 
 	needUpdateAfterCreate := false
 
+	if len(aircs) > 0 {
+		needUpdateAfterCreate = true
+	}
+
 	// For now PSC based cluster don't support `enable_private_endpoint` on `create`, but only on `update` API call.
 	// If cluster is PSC based and enable_private_endpoint is set to true we will ignore it on `create` call and update cluster right after creation.
 	enablePrivateEndpointPSCCluster := isEnablePrivateEndpointPSCCluster(cluster)
@@ -3014,6 +3039,13 @@ func resourceContainerClusterCreate(d *schema.ResourceData, meta interface{}) er
 			}
 			update.ForceSendFields = append(update.ForceSendFields, "DesiredAddonsConfig.GcePersistentDiskCsiDriverConfig.Enabled")
 		}
+
+		if len(aircs) > 0 {
+			update.DesiredAdditionalIpRangesConfig = &container.DesiredAdditionalIPRangesConfig{
+				AdditionalIpRangesConfigs: aircs,
+			}
+		}
+
 		req := &container.UpdateClusterRequest{Update: update}
 
 		err = transport_tpg.Retry(transport_tpg.RetryOptions{
@@ -4214,6 +4246,30 @@ func resourceContainerClusterUpdate(d *schema.ResourceData, meta interface{}) er
 		log.Printf("[INFO] GKE cluster %s's AdditionalPodRangesConfig has been updated", d.Id())
 	}
 
+	if d.HasChange("ip_allocation_policy.0.additional_ip_ranges_config") {
+		c := d.Get("ip_allocation_policy.0.additional_ip_ranges_config")
+		aircs, err := expandAdditionalIpRangesConfigs(c, d, config)
+		if err != nil {
+			return err
+		}
+
+		req := &container.UpdateClusterRequest{
+			Update: &container.ClusterUpdate{
+				DesiredAdditionalIpRangesConfig: &container.DesiredAdditionalIPRangesConfig{
+					AdditionalIpRangesConfigs: aircs,
+				},
+			},
+		}
+
+		updateF := updateFunc(req, "updating AdditionalIpRangesConfig")
+		// Call update serially.
+		if err := transport_tpg.LockedCall(lockKey, updateF); err != nil {
+			return err
+		}
+
+		log.Printf("[INFO] GKE cluster %s's AdditionalIpRangesConfig has been updated", d.Id())
+	}
+
 	if n, ok := d.GetOk("node_pool.#"); ok {
 		for i := 0; i < n.(int); i++ {
 			nodePoolInfo, err := extractNodePoolInformationFromCluster(d, config, clusterName)
@@ -5366,23 +5422,66 @@ func expandPodCidrOverprovisionConfig(configured interface{}) *container.PodCIDR
 	}
 }
 
-func expandIPAllocationPolicy(configured interface{}, networkingMode string, autopilot bool) (*container.IPAllocationPolicy, error) {
+func expandPodIpv4RangeNames(configured interface{}) []string {
+	l := configured.([]interface{})
+	if len(l) == 0 || l[0] == nil {
+		return nil
+	}
+	var ranges []string
+	for _, rawRange := range l {
+		ranges = append(ranges, rawRange.(string))
+	}
+	return ranges
+}
+
+func expandAdditionalIpRangesConfigs(configured interface{}, d *schema.ResourceData, c *transport_tpg.Config) ([]*container.AdditionalIPRangesConfig, error) {
+	l := configured.([]interface{})
+	if len(l) == 0 || l[0] == nil {
+		return nil, nil
+	}
+	var additionalIpRangesConfig []*container.AdditionalIPRangesConfig
+	for _, rawConfig := range l {
+		config := rawConfig.(map[string]interface{})
+		subnetwork, err := tpgresource.ParseSubnetworkFieldValue(config["subnetwork"].(string), d, c)
+		if err != nil {
+			return nil, err
+		}
+		additionalIpRangesConfig = append(additionalIpRangesConfig, &container.AdditionalIPRangesConfig{
+			Subnetwork:        subnetwork.RelativeLink(),
+			PodIpv4RangeNames: expandPodIpv4RangeNames(config["pod_ipv4_range_names"]),
+		})
+	}
+
+	return additionalIpRangesConfig, nil
+}
+
+func expandIPAllocationPolicy(configured interface{}, d *schema.ResourceData, networkingMode string, autopilot bool, c *transport_tpg.Config) (*container.IPAllocationPolicy, []*container.AdditionalIPRangesConfig, error) {
 	l := configured.([]interface{})
 	if len(l) == 0 || l[0] == nil {
 		if networkingMode == "VPC_NATIVE" {
-			return nil, nil
+			return nil, nil, nil
 		}
 		return &container.IPAllocationPolicy{
 			UseIpAliases:    false,
 			UseRoutes:       true,
 			StackType:       "IPV4",
 			ForceSendFields: []string{"UseIpAliases"},
-		}, nil
+		}, nil, nil
 	}
 
 	config := l[0].(map[string]interface{})
 	stackType := config["stack_type"].(string)
 
+	// We expand and return additional_ip_ranges_config separately because
+	// this field is OUTPUT_ONLY for ClusterCreate RPCs. Instead, during the
+	// Terraform Create flow, we follow the CreateCluster (without
+	// additional_ip_ranges_config populated) with an UpdateCluster (_with_
+	// additional_ip_ranges_config populated).
+	additionalIpRangesConfigs, err := expandAdditionalIpRangesConfigs(config["additional_ip_ranges_config"], d, c)
+	if err != nil {
+		return nil, nil, err
+	}
+
 	return &container.IPAllocationPolicy{
 		UseIpAliases:               networkingMode == "VPC_NATIVE" || networkingMode == "",
 		ClusterIpv4CidrBlock:       config["cluster_ipv4_cidr_block"].(string),
@@ -5393,7 +5492,7 @@ func expandIPAllocationPolicy(configured interface{}, networkingMode string, aut
 		UseRoutes:                  networkingMode == "ROUTES",
 		StackType:                  stackType,
 		PodCidrOverprovisionConfig: expandPodCidrOverprovisionConfig(config["pod_cidr_overprovision_config"]),
-	}, nil
+	}, additionalIpRangesConfigs, nil
 }
 
 func expandMaintenancePolicy(d *schema.ResourceData, meta interface{}) *container.MaintenancePolicy {
@@ -6980,6 +7079,23 @@ func flattenPodCidrOverprovisionConfig(c *container.PodCIDROverprovisionConfig)
 	}
 }
 
+func flattenAdditionalIpRangesConfigs(c []*container.AdditionalIPRangesConfig) []map[string]interface{} {
+	if len(c) == 0 {
+		return nil
+	}
+
+	var outRanges []map[string]interface{}
+	for _, rangeConfig := range c {
+		outRangeConfig := map[string]interface{}{
+			"subnetwork":           rangeConfig.Subnetwork,
+			"pod_ipv4_range_names": rangeConfig.PodIpv4RangeNames,
+		}
+		outRanges = append(outRanges, outRangeConfig)
+	}
+
+	return outRanges
+}
+
 func flattenIPAllocationPolicy(c *container.Cluster, d *schema.ResourceData, config *transport_tpg.Config) ([]map[string]interface{}, error) {
 	// If IP aliasing isn't enabled, none of the values in this block can be set.
 	if c == nil || c.IpAllocationPolicy == nil || !c.IpAllocationPolicy.UseIpAliases {
@@ -7010,6 +7126,7 @@ func flattenIPAllocationPolicy(c *container.Cluster, d *schema.ResourceData, con
 			"stack_type":                    p.StackType,
 			"pod_cidr_overprovision_config": flattenPodCidrOverprovisionConfig(p.PodCidrOverprovisionConfig),
 			"additional_pod_ranges_config":  flattenAdditionalPodRangesConfig(c.IpAllocationPolicy),
+			"additional_ip_ranges_config":   flattenAdditionalIpRangesConfigs(p.AdditionalIpRangesConfigs),
 		},
 	}, nil
 }

google-beta/services/container/resource_container_cluster_meta.yaml

@@ -148,6 +148,8 @@ fields:
   - field: 'identity_service_config.enabled'
   - field: 'initial_node_count'
   - field: 'ip_allocation_policy.additional_pod_ranges_config.pod_range_names'
+  - field: 'ip_allocation_policy.additional_ip_ranges_config.subnetwork'
+  - field: 'ip_allocation_policy.additional_ip_ranges_config.pod_ipv4_range_names'
   - field: 'ip_allocation_policy.cluster_ipv4_cidr_block'
   - field: 'ip_allocation_policy.cluster_secondary_range_name'
   - field: 'ip_allocation_policy.pod_cidr_overprovision_config.disabled'

google-beta/services/container/resource_container_cluster_test.go

@@ -14011,6 +14011,88 @@ resource "google_container_cluster" "primary" {
 `, name, networkName, subnetworkName, config)
 }
 
+func TestAccContainerCluster_additional_ip_ranges_config_on_create(t *testing.T) {
+	t.Parallel()
+
+	clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10))
+	acctest.VcrTest(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+		CheckDestroy:             testAccCheckContainerClusterDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccContainerCluster_additional_ip_ranges_config(clusterName, 2, 2),
+			},
+			{
+				ResourceName:            "google_container_cluster.primary",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"deletion_protection"},
+				Check:                   resource.TestCheckResourceAttrSet("google_container_cluster.primary", "node_pool.0.network_config.subnetwork"),
+			},
+		},
+	})
+}
+
+func TestAccContainerCluster_additional_ip_ranges_config_on_update(t *testing.T) {
+	t.Parallel()
+
+	clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10))
+	acctest.VcrTest(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+		CheckDestroy:             testAccCheckContainerClusterDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccContainerCluster_additional_ip_ranges_config(clusterName, 0, 0),
+			},
+			{
+				ResourceName:            "google_container_cluster.primary",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"deletion_protection"},
+				Check:                   resource.TestCheckResourceAttrSet("google_container_cluster.primary", "node_pool.0.network_config.subnetwork"),
+			},
+			{
+				Config: testAccContainerCluster_additional_ip_ranges_config(clusterName, 1, 1),
+			},
+			{
+				ResourceName:            "google_container_cluster.primary",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"deletion_protection"},
+			},
+			{
+				Config: testAccContainerCluster_additional_ip_ranges_config(clusterName, 0, 0),
+			},
+			{
+				ResourceName:            "google_container_cluster.primary",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"deletion_protection"},
+			},
+			{
+				Config: testAccContainerCluster_additional_ip_ranges_config(clusterName, 2, 2),
+			},
+			{
+				ResourceName:            "google_container_cluster.primary",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"deletion_protection"},
+			},
+			{
+				Config: testAccContainerCluster_additional_ip_ranges_config(clusterName, 0, 0),
+			},
+			{
+				ResourceName:            "google_container_cluster.primary",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"deletion_protection"},
+			},
+		},
+	})
+}
+
 func TestAccContainerCluster_withAnonymousAuthenticationConfig(t *testing.T) {
 	t.Parallel()
 
@@ -14051,6 +14133,89 @@ func TestAccContainerCluster_withAnonymousAuthenticationConfig(t *testing.T) {
 	})
 }
 
+func testAccContainerCluster_additional_ip_ranges_config(name string, additionalSubnetCount int, secondaryRangeCount int) string {
+	var subnetStr string
+	var additionalIpRangesStr string
+	cumulativeRangeIndex := 0
+	for subnetIndex := 0; subnetIndex < additionalSubnetCount; subnetIndex++ {
+		var secondaryRangeStr string
+		var podIpv4RangeStr string
+		for rangeIndex := 0; rangeIndex < secondaryRangeCount; rangeIndex++ {
+			secondaryRangeStr += fmt.Sprintf(`
+                            secondary_ip_range {
+                              range_name = "range-%d"
+                              ip_cidr_range = "10.0.%d.0/24"
+                            }
+                          `, cumulativeRangeIndex, cumulativeRangeIndex)
+
+			podIpv4RangeStr += fmt.Sprintf("google_compute_subnetwork.extra_%d.secondary_ip_range[%d].range_name", subnetIndex, rangeIndex)
+			if rangeIndex != secondaryRangeCount-1 {
+				podIpv4RangeStr += ", "
+			}
+			cumulativeRangeIndex++
+		}
+
+		subnetStr += fmt.Sprintf(`
+                           resource "google_compute_subnetwork" "extra_%d" {
+                             ip_cidr_range = "10.1.%d.0/24"
+                             name = "tf-test-subnet-%d"
+                             network = google_compute_network.main.self_link
+                             region = "us-central1"
+                             %s
+                           }
+                         `, subnetIndex, subnetIndex, subnetIndex, secondaryRangeStr)
+
+		additionalIpRangesStr += fmt.Sprintf(`
+			additional_ip_ranges_config {
+				subnetwork  = google_compute_subnetwork.extra_%d.id
+				pod_ipv4_range_names = [%s]
+			}
+		`, subnetIndex, podIpv4RangeStr)
+	}
+
+	return fmt.Sprintf(`
+	resource "google_compute_network" "main" {
+	  name                    = "%s"
+	  auto_create_subnetworks = false
+	}
+
+	resource "google_compute_subnetwork" "main" {
+	  ip_cidr_range = "10.2.0.0/24"
+	  name          = "main"
+	  network       = google_compute_network.main.self_link
+	  region        = "us-central1"
+
+	  secondary_ip_range {
+	    range_name    = "services"
+	    ip_cidr_range = "10.3.0.0/16"
+	  }
+
+	  secondary_ip_range {
+	    range_name    = "pods"
+	    ip_cidr_range = "10.4.0.0/16"
+	  }
+	}
+
+        %s
+
+	resource "google_container_cluster" "primary" {
+	  name     = "%s"
+	  location = "us-central1-a"
+	  network    = google_compute_network.main.name
+	  subnetwork = google_compute_subnetwork.main.name
+          initial_node_count = 1
+
+	  ip_allocation_policy {
+	    cluster_secondary_range_name  = "pods"
+	    services_secondary_range_name = "services"
+	    %s
+	  }
+
+	  deletion_protection = false
+	}
+	`, name, subnetStr, name, additionalIpRangesStr)
+}
+
 func testAccContainerCluster_withAnonymousAuthenticationConfig(name, networkName, subnetworkName string, mode string) string {
 	return fmt.Sprintf(`
 resource "google_container_cluster" "primary" {