google_datastream_stream: Add support for projectID field in SourceHierarchyDatasets (#14858)

Co-authored-by: Nick Elliot <[email protected]>

Author: benjamin-maynard

mmv1/products/datastream/Stream.yaml

@@ -180,6 +180,26 @@ examples:
     external_providers: ["random", "time"]
       # Random provider
     skip_vcr: true
+  - name: 'datastream_stream_bigquery_cross_project_source_hierachy'
+    primary_resource_id: 'default'
+    vars:
+      stream_id: 'my-stream'
+      private_connection_id: 'my-connection'
+      network_name: 'my-network'
+      source_connection_profile_id: 'source-profile'
+      database_instance_name: 'my-instance'
+      deletion_protection: 'true'
+      destination_connection_profile_id: 'destination-profile'
+    test_env_vars:
+      org_id: 'ORG_ID'
+      billing_account: 'BILLING_ACCT'
+    test_vars_overrides:
+      'deletion_protection': 'false'
+    oics_vars_overrides:
+      'deletion_protection': 'false'
+    external_providers: ["random", "time"]
+      # Random provider
+    skip_vcr: true
   - name: 'datastream_stream_bigquery_append_only'
     primary_resource_id: 'default'
     vars:
@@ -1337,6 +1357,10 @@ properties:
                       encryption key. i.e. projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{cryptoKey}.
                       See https://cloud.google.com/bigquery/docs/customer-managed-encryption for more information.
                     immutable: true
+              - name: 'projectId'
+                type: string
+                description: |
+                  Optional. The project id of the BigQuery dataset. If not specified, the project will be inferred from the stream resource.
           - name: 'blmtConfig'
             type: NestedObject
             description: 'BigLake Managed Tables configuration for BigQuery streams.'

mmv1/templates/terraform/examples/datastream_stream_bigquery_cross_project_source_hierachy.tf.tmpl

@@ -0,0 +1,129 @@
+data "google_project" "project" {
+}
+
+resource "google_project" "cross-project-dataset" {
+  project_id      = "tf-test%{random_suffix}"
+  name            = "tf-test%{random_suffix}"
+  org_id          = "{{index $.TestEnvVars "org_id"}}"
+  billing_account = "{{index $.TestEnvVars "billing_account"}}"
+  deletion_policy = "DELETE"
+}
+
+resource "time_sleep" "wait_60_seconds" {
+  create_duration = "60s"
+  depends_on = [google_project.cross-project-dataset]
+}
+
+resource "google_project_service" "bigquery" {
+  project = google_project.cross-project-dataset.project_id
+  service = "bigquery.googleapis.com"
+  disable_on_destroy = false
+  depends_on = [time_sleep.wait_60_seconds]
+}
+
+resource "google_project_iam_member" "datastream_bigquery_admin" {
+  project = google_project.cross-project-dataset.project_id
+  role    = "roles/bigquery.admin"
+  member  = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datastream.iam.gserviceaccount.com"
+  depends_on = [time_sleep.wait_60_seconds]
+}
+
+resource "google_sql_database_instance" "instance" {
+    name             = "{{index $.Vars "database_instance_name"}}"
+    database_version = "MYSQL_8_0"
+    region           = "us-central1"
+    settings {
+        tier = "db-f1-micro"
+        backup_configuration {
+            enabled            = true
+            binary_log_enabled = true
+        }
+
+        ip_configuration {
+
+            // Datastream IPs will vary by region.
+            authorized_networks {
+                value = "34.71.242.81"
+            }
+
+            authorized_networks {
+                value = "34.72.28.29"
+            }
+
+            authorized_networks {
+                value = "34.67.6.157"
+            }
+
+            authorized_networks {
+                value = "34.67.234.134"
+            }
+
+            authorized_networks {
+                value = "34.72.239.218"
+            }
+        }
+    }
+
+    deletion_protection  = {{index $.Vars "deletion_protection"}}
+}
+
+resource "google_sql_database" "db" {
+    instance = google_sql_database_instance.instance.name
+    name     = "db"
+}
+
+resource "random_password" "pwd" {
+    length = 16
+    special = false
+}
+
+resource "google_sql_user" "user" {
+    name     = "user"
+    instance = google_sql_database_instance.instance.name
+    host     = "%"
+    password = random_password.pwd.result
+}
+
+resource "google_datastream_connection_profile" "source_connection_profile" {
+    display_name          = "Source connection profile"
+    location              = "us-central1"
+    connection_profile_id = "{{index $.Vars "source_connection_profile_id"}}"
+
+    mysql_profile {
+        hostname = google_sql_database_instance.instance.public_ip_address
+        username = google_sql_user.user.name
+        password = google_sql_user.user.password
+    }
+}
+
+resource "google_datastream_connection_profile" "destination_connection_profile" {
+    display_name          = "Connection profile"
+    location              = "us-central1"
+    connection_profile_id = "{{index $.Vars "destination_connection_profile_id"}}"
+
+    bigquery_profile {}
+}
+
+resource "google_datastream_stream" "{{$.PrimaryResourceId}}" {
+    stream_id = "{{index $.Vars "stream_id"}}"
+    location = "us-central1"
+    display_name = "my stream"
+    source_config {
+        source_connection_profile = google_datastream_connection_profile.source_connection_profile.id
+        mysql_source_config {}
+    }
+    destination_config {
+        destination_connection_profile = google_datastream_connection_profile.destination_connection_profile.id
+        bigquery_destination_config {
+            source_hierarchy_datasets {
+                dataset_template {
+                    location = "us-central1"
+                }
+                project_id = google_project.cross-project-dataset.project_id
+            }
+        }
+    }
+
+    backfill_none {
+    }
+}