google_datastream_stream: Add support for projectID field in SourceHierarchyDatasets (#14858) (#10704)

[upstream:9b1a11e9d9d58895df0acf835427cd7e23a56abc]

Signed-off-by: Modular Magician <[email protected]>

Author: modular-magician

.changelog/14858.txt

@@ -0,0 +1,58 @@
+```release-note:enhancement
+datastream: added `destination_config.bigquery_destination_config.source_hierarchy_datasets.project_id` field to `google_datastream_stream` resource
+```
+
+google_datastream_stream: Add support for projectID field in SourceHierarchyDatasets. 
+
+Also added a new test in [`mmv1/templates/terraform/examples/datastream_stream_bigquery_cross_project_source_hierachy.tf.tmpl`](https://github.com/benjamin-maynard/magic-modules/blob/c1aaf45e21117e93ccbd5c25461efc2170b7b49e/mmv1/templates/terraform/examples/datastream_stream_bigquery_cross_project_source_hierachy.tf.tmpl).
+
+I also had to add `default_from_api: true` to `binaryLogPosition` (the default) as **all** of the BigQuery tests were failing with a diff-after-apply. Unrelated to my changes¹.
+
+I believe this is safe and all of the tests pass² (although the [blmt test is skipped](https://github.com/benjamin-maynard/magic-modules/blob/c1aaf45e21117e93ccbd5c25461efc2170b7b49e/mmv1/products/datastream/Stream.yaml#L234-L235)), but please let me know if any concerns.
+
+
+[1]
+```
+Step 1/2 error: After applying this test step, the plan was not empty.
+        stdout:
+        Terraform used the selected providers to generate the following execution
+        plan. Resource actions are indicated with the following symbols:
+          ~ update in-place
+        Terraform will perform the following actions:   
+          # google_datastream_stream.default will be updated in-place
+          ~ resource "google_datastream_stream" "default" {
+                id                              = "projects/platform-dev-8bvnw/locations/us-central1/streams/tf-test-my-streammpuoq3q4ky"
+                name                            = "projects/platform-dev-8bvnw/locations/us-central1/streams/tf-test-my-streammpuoq3q4ky"
+                # (10 unchanged attributes hidden)
+              ~ source_config {
+                    # (1 unchanged attribute hidden)
+                  ~ mysql_source_config {
+                        # (2 unchanged attributes hidden)
+                      - binary_log_position {}
+                    }
+                # (2 unchanged blocks hidden)
+            }      
+        Plan: 0 to add, 1 to change, 0 to destroy.
+```
+
+[2]
+```
+benmaynard$ make testacc TEST=./google/services/datastream TESTARGS='-run=TestAccDatastreamStream_datastreamStreamBigquery'
+TF_ACC=1 TF_SCHEMA_PANIC_ON_ERROR=1 go test ./google/services/datastream -v -run=TestAccDatastreamStream_datastreamStreamBigquery -timeout 240m -ldflags="-X=github.com/hashicorp/terraform-provider-google/version.ProviderVersion=acc"
+=== RUN   TestAccDatastreamStream_datastreamStreamBigqueryExample
+=== PAUSE TestAccDatastreamStream_datastreamStreamBigqueryExample
+=== RUN   TestAccDatastreamStream_datastreamStreamBigqueryCrossProjectSourceHierachyExample
+=== PAUSE TestAccDatastreamStream_datastreamStreamBigqueryCrossProjectSourceHierachyExample
+=== RUN   TestAccDatastreamStream_datastreamStreamBigqueryAppendOnlyExample
+=== PAUSE TestAccDatastreamStream_datastreamStreamBigqueryAppendOnlyExample
+=== CONT  TestAccDatastreamStream_datastreamStreamBigqueryExample
+=== CONT  TestAccDatastreamStream_datastreamStreamBigqueryAppendOnlyExample
+=== CONT  TestAccDatastreamStream_datastreamStreamBigqueryCrossProjectSourceHierachyExample
+2025/08/14 14:48:16 [INFO] Authenticating using DefaultClient...
+2025/08/14 14:48:16 [INFO]   -- Scopes: [https://www.googleapis.com/auth/cloud-platform https://www.googleapis.com/auth/userinfo.email]
+--- PASS: TestAccDatastreamStream_datastreamStreamBigqueryAppendOnlyExample (1369.56s)
+--- PASS: TestAccDatastreamStream_datastreamStreamBigqueryExample (1420.77s)
+--- PASS: TestAccDatastreamStream_datastreamStreamBigqueryCrossProjectSourceHierachyExample (1503.81s)
+PASS
+ok      github.com/hashicorp/terraform-provider-google/google/services/datastream       1505.460s
+```

google-beta/services/datastream/resource_datastream_stream.go

@@ -274,6 +274,11 @@ See https://cloud.google.com/bigquery/docs/customer-managed-encryption for more
 														},
 													},
 												},
+												"project_id": {
+													Type:        schema.TypeString,
+													Optional:    true,
+													Description: `Optional. The project id of the BigQuery dataset. If not specified, the project will be inferred from the stream resource.`,
+												},
 											},
 										},
 										ExactlyOneOf: []string{"destination_config.0.bigquery_destination_config.0.single_target_dataset", "destination_config.0.bigquery_destination_config.0.source_hierarchy_datasets"},
@@ -4199,6 +4204,8 @@ func flattenDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHier
 	transformed := make(map[string]interface{})
 	transformed["dataset_template"] =
 		flattenDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsDatasetTemplate(original["datasetTemplate"], d, config)
+	transformed["project_id"] =
+		flattenDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsProjectId(original["projectId"], d, config)
 	return []interface{}{transformed}
 }
 func flattenDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsDatasetTemplate(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
@@ -4230,6 +4237,10 @@ func flattenDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHier
 	return v
 }
 
+func flattenDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsProjectId(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
+	return v
+}
+
 func flattenDatastreamStreamDestinationConfigBigqueryDestinationConfigBlmtConfig(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
 	if v == nil {
 		return nil
@@ -7414,6 +7425,13 @@ func expandDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHiera
 		transformed["datasetTemplate"] = transformedDatasetTemplate
 	}
 
+	transformedProjectId, err := expandDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsProjectId(original["project_id"], d, config)
+	if err != nil {
+		return nil, err
+	} else if val := reflect.ValueOf(transformedProjectId); val.IsValid() && !tpgresource.IsEmptyValue(val) {
+		transformed["projectId"] = transformedProjectId
+	}
+
 	return transformed, nil
 }
 
@@ -7462,6 +7480,10 @@ func expandDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHiera
 	return v, nil
 }
 
+func expandDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsProjectId(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
+	return v, nil
+}
+
 func expandDatastreamStreamDestinationConfigBigqueryDestinationConfigBlmtConfig(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
 	l := v.([]interface{})
 	if len(l) == 0 || l[0] == nil {

google-beta/services/datastream/resource_datastream_stream_generated_meta.yaml

@@ -65,6 +65,7 @@ fields:
   - field: 'destination_config.bigquery_destination_config.source_hierarchy_datasets.dataset_template.dataset_id_prefix'
   - field: 'destination_config.bigquery_destination_config.source_hierarchy_datasets.dataset_template.kms_key_name'
   - field: 'destination_config.bigquery_destination_config.source_hierarchy_datasets.dataset_template.location'
+  - field: 'destination_config.bigquery_destination_config.source_hierarchy_datasets.project_id'
   - field: 'destination_config.destination_connection_profile'
   - field: 'destination_config.gcs_destination_config.avro_file_format'
   - field: 'destination_config.gcs_destination_config.file_rotation_interval'

google-beta/services/datastream/resource_datastream_stream_generated_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/hashicorp/terraform-plugin-testing/terraform"
 
 	"github.com/hashicorp/terraform-provider-google-beta/google-beta/acctest"
+	"github.com/hashicorp/terraform-provider-google-beta/google-beta/envvar"
 	"github.com/hashicorp/terraform-provider-google-beta/google-beta/tpgresource"
 	transport_tpg "github.com/hashicorp/terraform-provider-google-beta/google-beta/transport"
 )
@@ -719,6 +720,173 @@ resource "google_datastream_stream" "default" {
 `, context)
 }
 
+func TestAccDatastreamStream_datastreamStreamBigqueryCrossProjectSourceHierachyExample(t *testing.T) {
+	acctest.SkipIfVcr(t)
+	t.Parallel()
+
+	context := map[string]interface{}{
+		"billing_account":     envvar.GetTestBillingAccountFromEnv(t),
+		"org_id":              envvar.GetTestOrgFromEnv(t),
+		"deletion_protection": false,
+		"random_suffix":       acctest.RandString(t, 10),
+	}
+
+	acctest.VcrTest(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+		ExternalProviders: map[string]resource.ExternalProvider{
+			"random": {},
+			"time":   {},
+		},
+		CheckDestroy: testAccCheckDatastreamStreamDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccDatastreamStream_datastreamStreamBigqueryCrossProjectSourceHierachyExample(context),
+			},
+			{
+				ResourceName:            "google_datastream_stream.default",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"create_without_validation", "labels", "location", "stream_id", "terraform_labels"},
+			},
+		},
+	})
+}
+
+func testAccDatastreamStream_datastreamStreamBigqueryCrossProjectSourceHierachyExample(context map[string]interface{}) string {
+	return acctest.Nprintf(`
+data "google_project" "project" {
+}
+
+resource "google_project" "cross-project-dataset" {
+  project_id      = "tf-test%{random_suffix}"
+  name            = "tf-test%{random_suffix}"
+  org_id          = "%{org_id}"
+  billing_account = "%{billing_account}"
+  deletion_policy = "DELETE"
+}
+
+resource "time_sleep" "wait_60_seconds" {
+  create_duration = "60s"
+  depends_on = [google_project.cross-project-dataset]
+}
+
+resource "google_project_service" "bigquery" {
+  project = google_project.cross-project-dataset.project_id
+  service = "bigquery.googleapis.com"
+  disable_on_destroy = false
+  depends_on = [time_sleep.wait_60_seconds]
+}
+
+resource "google_project_iam_member" "datastream_bigquery_admin" {
+  project = google_project.cross-project-dataset.project_id
+  role    = "roles/bigquery.admin"
+  member  = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datastream.iam.gserviceaccount.com"
+  depends_on = [time_sleep.wait_60_seconds]
+}
+
+resource "google_sql_database_instance" "instance" {
+    name             = "tf-test-my-instance%{random_suffix}"
+    database_version = "MYSQL_8_0"
+    region           = "us-central1"
+    settings {
+        tier = "db-f1-micro"
+        backup_configuration {
+            enabled            = true
+            binary_log_enabled = true
+        }
+
+        ip_configuration {
+
+            // Datastream IPs will vary by region.
+            authorized_networks {
+                value = "34.71.242.81"
+            }
+
+            authorized_networks {
+                value = "34.72.28.29"
+            }
+
+            authorized_networks {
+                value = "34.67.6.157"
+            }
+
+            authorized_networks {
+                value = "34.67.234.134"
+            }
+
+            authorized_networks {
+                value = "34.72.239.218"
+            }
+        }
+    }
+
+    deletion_protection  = %{deletion_protection}
+}
+
+resource "google_sql_database" "db" {
+    instance = google_sql_database_instance.instance.name
+    name     = "db"
+}
+
+resource "random_password" "pwd" {
+    length = 16
+    special = false
+}
+
+resource "google_sql_user" "user" {
+    name     = "user"
+    instance = google_sql_database_instance.instance.name
+    host     = "%"
+    password = random_password.pwd.result
+}
+
+resource "google_datastream_connection_profile" "source_connection_profile" {
+    display_name          = "Source connection profile"
+    location              = "us-central1"
+    connection_profile_id = "tf-test-source-profile%{random_suffix}"
+
+    mysql_profile {
+        hostname = google_sql_database_instance.instance.public_ip_address
+        username = google_sql_user.user.name
+        password = google_sql_user.user.password
+    }
+}
+
+resource "google_datastream_connection_profile" "destination_connection_profile" {
+    display_name          = "Connection profile"
+    location              = "us-central1"
+    connection_profile_id = "tf-test-destination-profile%{random_suffix}"
+
+    bigquery_profile {}
+}
+
+resource "google_datastream_stream" "default" {
+    stream_id = "tf-test-my-stream%{random_suffix}"
+    location = "us-central1"
+    display_name = "my stream"
+    source_config {
+        source_connection_profile = google_datastream_connection_profile.source_connection_profile.id
+        mysql_source_config {}
+    }
+    destination_config {
+        destination_connection_profile = google_datastream_connection_profile.destination_connection_profile.id
+        bigquery_destination_config {
+            source_hierarchy_datasets {
+                dataset_template {
+                    location = "us-central1"
+                }
+                project_id = google_project.cross-project-dataset.project_id
+            }
+        }
+    }
+
+    backfill_none {
+    }
+}
+`, context)
+}
+
 func TestAccDatastreamStream_datastreamStreamBigqueryAppendOnlyExample(t *testing.T) {
 	acctest.SkipIfVcr(t)
 	t.Parallel()