New fields: enable_private_environment, enable_private_builds_only (#9689) (#6870)

* add enable_private_environment and enable_private_builds_only configs

* add tests

* correction in config name

* remove unnecessary checks

* case that PrivateEnvironmentConfig is nil

* add edge case tests for composer 3

* make new fields beta only

* add docs

* correct merge mistake

* avoid false negative for enablePrivateEnvironment

* check image version to avoid overriding EnablePrivateEnvironment

* minor fixes

* simplify code
[upstream:8df5516b9f3d17877b0cd046563352c0ded6c2a1]

Signed-off-by: Modular Magician <[email protected]>

Author: modular-magician

.changelog/9689.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+composer: added fields `enable_private_environment` and `enable_private_builds_only` to `google_composer_environment`
+```

google-beta/services/composer/resource_composer_environment.go

@@ -70,6 +70,8 @@ var (
 		"config.0.environment_size",
 		"config.0.master_authorized_networks_config",
 		"config.0.resilience_mode",
+		"config.0.enable_private_environment",
+		"config.0.enable_private_builds_only",
 	}
 
 	recoveryConfigKeys = []string{
@@ -559,6 +561,22 @@ func ResourceComposerEnvironment() *schema.Resource {
 								},
 							},
 						},
+						"enable_private_environment": {
+							Type:         schema.TypeBool,
+							Computed:     true,
+							Optional:     true,
+							ForceNew:     false,
+							AtLeastOneOf: composerConfigKeys,
+							Description:  `Optional. If true, a private Composer environment will be created.`,
+						},
+						"enable_private_builds_only": {
+							Type:         schema.TypeBool,
+							Computed:     true,
+							Optional:     true,
+							ForceNew:     false,
+							AtLeastOneOf: composerConfigKeys,
+							Description:  `Optional. If true, builds performed during operations that install Python packages have only private connectivity to Google services. If false, the builds also have access to the internet.`,
+						},
 						"web_server_network_access_control": {
 							Type:         schema.TypeList,
 							Optional:     true,
@@ -1221,6 +1239,33 @@ func resourceComposerEnvironmentUpdate(d *schema.ResourceData, meta interface{})
 			}
 		}
 
+		if d.HasChange("config.0.enable_private_environment") {
+			patchObj := &composer.Environment{
+				Config: &composer.EnvironmentConfig{
+					PrivateEnvironmentConfig: &composer.PrivateEnvironmentConfig{},
+				},
+			}
+			if config != nil && config.PrivateEnvironmentConfig != nil {
+				patchObj.Config.PrivateEnvironmentConfig.EnablePrivateEnvironment = config.PrivateEnvironmentConfig.EnablePrivateEnvironment
+			}
+			err = resourceComposerEnvironmentPatchField("config.PrivateEnvironmentConfig.EnablePrivateEnvironment", userAgent, patchObj, d, tfConfig)
+			if err != nil {
+				return err
+			}
+		}
+
+		if d.HasChange("config.0.enable_private_builds_only") {
+			patchObj := &composer.Environment{
+				Config: &composer.EnvironmentConfig{
+					PrivateEnvironmentConfig: &composer.PrivateEnvironmentConfig{},
+				},
+			}
+			if config != nil && config.PrivateEnvironmentConfig != nil {
+				patchObj.Config.PrivateEnvironmentConfig.EnablePrivateBuildsOnly = config.PrivateEnvironmentConfig.EnablePrivateBuildsOnly
+			}
+			err = resourceComposerEnvironmentPatchField("config.PrivateEnvironmentConfig.EnablePrivateBuildsOnly", userAgent, patchObj, d, tfConfig)
+		}
+
 		if d.HasChange("config.0.software_config.0.web_server_plugins_mode") {
 			patchObj := &composer.Environment{
 				Config: &composer.EnvironmentConfig{
@@ -1476,6 +1521,8 @@ func flattenComposerEnvironmentConfig(envCfg *composer.EnvironmentConfig) interf
 	transformed["node_config"] = flattenComposerEnvironmentConfigNodeConfig(envCfg.NodeConfig)
 	transformed["software_config"] = flattenComposerEnvironmentConfigSoftwareConfig(envCfg.SoftwareConfig)
 	transformed["private_environment_config"] = flattenComposerEnvironmentConfigPrivateEnvironmentConfig(envCfg.PrivateEnvironmentConfig)
+	transformed["enable_private_environment"] = envCfg.PrivateEnvironmentConfig.EnablePrivateEnvironment
+	transformed["enable_private_builds_only"] = envCfg.PrivateEnvironmentConfig.EnablePrivateBuildsOnly
 	transformed["web_server_network_access_control"] = flattenComposerEnvironmentConfigWebServerNetworkAccessControl(envCfg.WebServerNetworkAccessControl)
 	transformed["database_config"] = flattenComposerEnvironmentConfigDatabaseConfig(envCfg.DatabaseConfig)
 	transformed["web_server_config"] = flattenComposerEnvironmentConfigWebServerConfig(envCfg.WebServerConfig)
@@ -1815,6 +1862,21 @@ func expandComposerEnvironmentConfig(v interface{}, d *schema.ResourceData, conf
 	}
 	transformed.PrivateEnvironmentConfig = transformedPrivateEnvironmentConfig
 
+	/*
+		config.enable_private_environment in terraform maps to
+		composer.PrivateEnvironmentConfig.EnablePrivateEnvironment in API.
+		Check image version to avoid overriding EnablePrivateEnvironment in case of other versions.
+	*/
+	if isComposer3(d, config) {
+		transformed.PrivateEnvironmentConfig = &composer.PrivateEnvironmentConfig{}
+		if enablePrivateEnvironmentRaw, ok := original["enable_private_environment"]; ok {
+			transformed.PrivateEnvironmentConfig.EnablePrivateEnvironment = enablePrivateEnvironmentRaw.(bool)
+		}
+		if enablePrivateBuildsOnlyRaw, ok := original["enable_private_builds_only"]; ok {
+			transformed.PrivateEnvironmentConfig.EnablePrivateBuildsOnly = enablePrivateBuildsOnlyRaw.(bool)
+		}
+	}
+
 	transformedWebServerNetworkAccessControl, err := expandComposerEnvironmentConfigWebServerNetworkAccessControl(original["web_server_network_access_control"], d, config)
 	if err != nil {
 		return nil, err
@@ -2688,3 +2750,8 @@ func versionsEqual(old, new string) (bool, error) {
 	}
 	return o.Equal(n), nil
 }
+
+func isComposer3(d *schema.ResourceData, config *transport_tpg.Config) bool {
+	image_version := d.Get("config.0.software_config.0.image_version").(string)
+	return strings.Contains(image_version, "composer-3")
+}

google-beta/services/composer/resource_composer_environment_test.go

@@ -1183,6 +1183,80 @@ func TestAccComposerEnvironmentComposer3_update(t *testing.T) {
 	})
 }
 
+// Checks Composer 3 specific updatable fields.
+func TestAccComposerEnvironmentComposer3_updateToEmpty(t *testing.T) {
+	t.Parallel()
+
+	envName := fmt.Sprintf("%s-%d", testComposerEnvironmentPrefix, acctest.RandInt(t))
+	network := fmt.Sprintf("%s-%d", testComposerNetworkPrefix, acctest.RandInt(t))
+	subnetwork := network + "-1"
+
+	acctest.VcrTest(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+		CheckDestroy:             testAccComposerEnvironmentDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccComposerEnvironmentComposer3_basic(envName, network, subnetwork),
+			},
+			{
+				Config: testAccComposerEnvironmentComposer3_empty(envName, network, subnetwork),
+			},
+			{
+				ResourceName:      "google_composer_environment.test",
+				ImportState:       true,
+				ImportStateVerify: true,
+			},
+			// This is a terrible clean-up step in order to get destroy to succeed,
+			// due to dangling firewall rules left by the Composer Environment blocking network deletion.
+			// TODO: Remove this check if firewall rules bug gets fixed by Composer.
+			{
+				PlanOnly:           true,
+				ExpectNonEmptyPlan: false,
+				Config:             testAccComposerEnvironmentComposer3_empty(envName, network, subnetwork),
+				Check:              testAccCheckClearComposerEnvironmentFirewalls(t, network),
+			},
+		},
+	})
+}
+
+// Checks Composer 3 specific updatable fields.
+func TestAccComposerEnvironmentComposer3_updateFromEmpty(t *testing.T) {
+	t.Parallel()
+
+	envName := fmt.Sprintf("%s-%d", testComposerEnvironmentPrefix, acctest.RandInt(t))
+	network := fmt.Sprintf("%s-%d", testComposerNetworkPrefix, acctest.RandInt(t))
+	subnetwork := network + "-1"
+
+	acctest.VcrTest(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+		CheckDestroy:             testAccComposerEnvironmentDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccComposerEnvironmentComposer3_empty(envName, network, subnetwork),
+			},
+			{
+				Config: testAccComposerEnvironmentComposer3_update(envName, network, subnetwork),
+			},
+			{
+				ResourceName:      "google_composer_environment.test",
+				ImportState:       true,
+				ImportStateVerify: true,
+			},
+			// This is a terrible clean-up step in order to get destroy to succeed,
+			// due to dangling firewall rules left by the Composer Environment blocking network deletion.
+			// TODO: Remove this check if firewall rules bug gets fixed by Composer.
+			{
+				PlanOnly:           true,
+				ExpectNonEmptyPlan: false,
+				Config:             testAccComposerEnvironmentComposer3_update(envName, network, subnetwork),
+				Check:              testAccCheckClearComposerEnvironmentFirewalls(t, network),
+			},
+		},
+	})
+}
+
 func testAccComposerEnvironment_customBucket(bucketName, envName, network, subnetwork string) string {
 	return fmt.Sprintf(`
 resource "google_storage_bucket" "test" {
@@ -2733,6 +2807,34 @@ resource "google_project_iam_member" "composer-worker" {
 `, environment, network, subnetwork, serviceAccount)
 }
 
+func testAccComposerEnvironmentComposer3_empty(name, network, subnetwork string) string {
+	return fmt.Sprintf(`
+resource "google_composer_environment" "test" {
+  name   = "%s"
+  region = "us-central1"
+  config {
+    software_config {
+      image_version = "composer-3-airflow-2"
+    }
+  }
+}
+
+// use a separate network to avoid conflicts with other tests running in parallel
+// that use the default network/subnet
+resource "google_compute_network" "test" {
+  name                    = "%s"
+  auto_create_subnetworks = false
+}
+
+resource "google_compute_subnetwork" "test" {
+  name          = "%s"
+  ip_cidr_range = "10.2.0.0/16"
+  region        = "us-central1"
+  network       = google_compute_network.test.self_link
+}
+`, name, network, subnetwork)
+}
+
 func testAccComposerEnvironmentComposer3_basic(name, network, subnetwork string) string {
 	return fmt.Sprintf(`
 resource "google_composer_environment" "test" {
@@ -2752,6 +2854,8 @@ resource "google_composer_environment" "test" {
         storage_gb   = 2
       }
     }
+    enable_private_environment = true
+    enable_private_builds_only = true
   }
 }
 
@@ -2777,20 +2881,22 @@ resource "google_composer_environment" "test" {
   name   = "%s"
   region = "us-central1"
   config {
+    node_config {
+      composer_internal_ipv4_cidr_block = "100.64.128.0/20"
+    }
     software_config {
       web_server_plugins_mode = "DISABLED"
       image_version = "composer-3-airflow-2"
     }
-    node_config {
-      composer_internal_ipv4_cidr_block = "100.64.128.0/20"
-    }
     workloads_config {
       dag_processor {
         cpu          = 2
         memory_gb    = 2
         storage_gb   = 1
       }
     }
+    enable_private_environment = false
+    enable_private_builds_only = false
   }
 }
 

website/docs/r/composer_environment.html.markdown

@@ -282,6 +282,15 @@ The following arguments are supported:
   (Optional)
   The configuration used for the Private IP Cloud Composer environment. Structure is [documented below](#nested_private_environment_config).
 
+* `enable_private_environment` -
+  (Optional, [Beta](https://terraform.io/docs/providers/google/guides/provider_versions.html), Cloud Composer 3 only)
+  If true, a private Composer environment will be created.
+
+* `enable_private_builds_only` -
+  (Optional, [Beta](https://terraform.io/docs/providers/google/guides/provider_versions.html), Cloud Composer 3 only)
+  If true, builds performed during operations that install Python packages have only private connectivity to Google services.
+  If false, the builds also have access to the internet.
+
 * `web_server_network_access_control` -
   The network-level access control policy for the Airflow web server.
   If unspecified, no network-level access restrictions are applied.