Add Pub/Sub Topic Source Support to Analytics Hub Listing (#14302) (#10219)

[upstream:a6026d82fbfc30149a93d5df6375514c077b1f5b]

Signed-off-by: Modular Magician <[email protected]>

Author: modular-magician

.changelog/14302.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+bigqueryanalyticshub: added `pubsub_topic` field to `google_bigquery_analytics_hub_listing` resource
+```

google-beta/services/bigqueryanalyticshub/resource_bigquery_analytics_hub_listing.go

@@ -56,9 +56,32 @@ func ResourceBigqueryAnalyticsHubListing() *schema.Resource {
 		),
 
 		Schema: map[string]*schema.Schema{
+			"data_exchange_id": {
+				Type:        schema.TypeString,
+				Required:    true,
+				ForceNew:    true,
+				Description: `The ID of the data exchange. Must contain only Unicode letters, numbers (0-9), underscores (_). Should not use characters that require URL-escaping, or characters outside of ASCII, spaces.`,
+			},
+			"display_name": {
+				Type:        schema.TypeString,
+				Required:    true,
+				Description: `Human-readable display name of the listing. The display name must contain only Unicode letters, numbers (0-9), underscores (_), dashes (-), spaces ( ), ampersands (&) and can't start or end with spaces.`,
+			},
+			"listing_id": {
+				Type:        schema.TypeString,
+				Required:    true,
+				ForceNew:    true,
+				Description: `The ID of the listing. Must contain only Unicode letters, numbers (0-9), underscores (_). Should not use characters that require URL-escaping, or characters outside of ASCII, spaces.`,
+			},
+			"location": {
+				Type:        schema.TypeString,
+				Required:    true,
+				ForceNew:    true,
+				Description: `The name of the location this data exchange listing.`,
+			},
 			"bigquery_dataset": {
 				Type:        schema.TypeList,
-				Required:    true,
+				Optional:    true,
 				ForceNew:    true,
 				Description: `Shared dataset i.e. BigQuery dataset source.`,
 				MaxItems:    1,
@@ -90,29 +113,7 @@ func ResourceBigqueryAnalyticsHubListing() *schema.Resource {
 						},
 					},
 				},
-			},
-			"data_exchange_id": {
-				Type:        schema.TypeString,
-				Required:    true,
-				ForceNew:    true,
-				Description: `The ID of the data exchange. Must contain only Unicode letters, numbers (0-9), underscores (_). Should not use characters that require URL-escaping, or characters outside of ASCII, spaces.`,
-			},
-			"display_name": {
-				Type:        schema.TypeString,
-				Required:    true,
-				Description: `Human-readable display name of the listing. The display name must contain only Unicode letters, numbers (0-9), underscores (_), dashes (-), spaces ( ), ampersands (&) and can't start or end with spaces.`,
-			},
-			"listing_id": {
-				Type:        schema.TypeString,
-				Required:    true,
-				ForceNew:    true,
-				Description: `The ID of the listing. Must contain only Unicode letters, numbers (0-9), underscores (_). Should not use characters that require URL-escaping, or characters outside of ASCII, spaces.`,
-			},
-			"location": {
-				Type:        schema.TypeString,
-				Required:    true,
-				ForceNew:    true,
-				Description: `The name of the location this data exchange listing.`,
+				ExactlyOneOf: []string{"pubsub_topic", "bigquery_dataset"},
 			},
 			"categories": {
 				Type:        schema.TypeList,
@@ -188,6 +189,34 @@ func ResourceBigqueryAnalyticsHubListing() *schema.Resource {
 					},
 				},
 			},
+			"pubsub_topic": {
+				Type:        schema.TypeList,
+				Optional:    true,
+				Description: `Pub/Sub topic source.`,
+				MaxItems:    1,
+				Elem: &schema.Resource{
+					Schema: map[string]*schema.Schema{
+						"topic": {
+							Type:             schema.TypeString,
+							Required:         true,
+							ForceNew:         true,
+							DiffSuppressFunc: tpgresource.ProjectNumberDiffSuppress,
+							Description:      `Resource name of the Pub/Sub topic source for this listing. e.g. projects/myproject/topics/topicId`,
+						},
+						"data_affinity_regions": {
+							Type:     schema.TypeSet,
+							Optional: true,
+							Description: `Region hint on where the data might be published. Data affinity regions are modifiable.
+See https://cloud.google.com/about/locations for full listing of possible Cloud regions.`,
+							Elem: &schema.Schema{
+								Type: schema.TypeString,
+							},
+							Set: schema.HashString,
+						},
+					},
+				},
+				ExactlyOneOf: []string{"pubsub_topic", "bigquery_dataset"},
+			},
 			"request_access": {
 				Type:        schema.TypeString,
 				Optional:    true,
@@ -302,6 +331,12 @@ func resourceBigqueryAnalyticsHubListingCreate(d *schema.ResourceData, meta inte
 	} else if v, ok := d.GetOkExists("bigquery_dataset"); !tpgresource.IsEmptyValue(reflect.ValueOf(bigqueryDatasetProp)) && (ok || !reflect.DeepEqual(v, bigqueryDatasetProp)) {
 		obj["bigqueryDataset"] = bigqueryDatasetProp
 	}
+	pubsubTopicProp, err := expandBigqueryAnalyticsHubListingPubsubTopic(d.Get("pubsub_topic"), d, config)
+	if err != nil {
+		return err
+	} else if v, ok := d.GetOkExists("pubsub_topic"); !tpgresource.IsEmptyValue(reflect.ValueOf(pubsubTopicProp)) && (ok || !reflect.DeepEqual(v, pubsubTopicProp)) {
+		obj["pubsubTopic"] = pubsubTopicProp
+	}
 	restrictedExportConfigProp, err := expandBigqueryAnalyticsHubListingRestrictedExportConfig(d.Get("restricted_export_config"), d, config)
 	if err != nil {
 		return err
@@ -436,6 +471,9 @@ func resourceBigqueryAnalyticsHubListingRead(d *schema.ResourceData, meta interf
 	if err := d.Set("bigquery_dataset", flattenBigqueryAnalyticsHubListingBigqueryDataset(res["bigqueryDataset"], d, config)); err != nil {
 		return fmt.Errorf("Error reading Listing: %s", err)
 	}
+	if err := d.Set("pubsub_topic", flattenBigqueryAnalyticsHubListingPubsubTopic(res["pubsubTopic"], d, config)); err != nil {
+		return fmt.Errorf("Error reading Listing: %s", err)
+	}
 	if err := d.Set("restricted_export_config", flattenBigqueryAnalyticsHubListingRestrictedExportConfig(res["restrictedExportConfig"], d, config)); err != nil {
 		return fmt.Errorf("Error reading Listing: %s", err)
 	}
@@ -516,6 +554,12 @@ func resourceBigqueryAnalyticsHubListingUpdate(d *schema.ResourceData, meta inte
 	} else if v, ok := d.GetOkExists("categories"); !tpgresource.IsEmptyValue(reflect.ValueOf(v)) && (ok || !reflect.DeepEqual(v, categoriesProp)) {
 		obj["categories"] = categoriesProp
 	}
+	pubsubTopicProp, err := expandBigqueryAnalyticsHubListingPubsubTopic(d.Get("pubsub_topic"), d, config)
+	if err != nil {
+		return err
+	} else if v, ok := d.GetOkExists("pubsub_topic"); !tpgresource.IsEmptyValue(reflect.ValueOf(v)) && (ok || !reflect.DeepEqual(v, pubsubTopicProp)) {
+		obj["pubsubTopic"] = pubsubTopicProp
+	}
 	restrictedExportConfigProp, err := expandBigqueryAnalyticsHubListingRestrictedExportConfig(d.Get("restricted_export_config"), d, config)
 	if err != nil {
 		return err
@@ -568,6 +612,10 @@ func resourceBigqueryAnalyticsHubListingUpdate(d *schema.ResourceData, meta inte
 		updateMask = append(updateMask, "categories")
 	}
 
+	if d.HasChange("pubsub_topic") {
+		updateMask = append(updateMask, "pubsubTopic")
+	}
+
 	if d.HasChange("restricted_export_config") {
 		updateMask = append(updateMask, "restrictedExportConfig")
 	}
@@ -577,6 +625,25 @@ func resourceBigqueryAnalyticsHubListingUpdate(d *schema.ResourceData, meta inte
 	if err != nil {
 		return err
 	}
+	if d.HasChange("pubsub_topic.0.data_affinity_regions") {
+		// Split URL into base and query parts
+		parts := strings.SplitN(url, "?", 2)
+		if len(parts) == 2 {
+			base := parts[0]
+			query := parts[1]
+
+			query = strings.ReplaceAll(query, "%2C", ",")
+			query = strings.ReplaceAll(query, "%2c", ",")
+
+			// Replace "pubsubTopic" with "pubsubTopic.dataAffinityRegions"
+			query = strings.ReplaceAll(query, "pubsubTopic", "pubsubTopic.dataAffinityRegions")
+
+			// Re-encode commas back
+			query = strings.ReplaceAll(query, ",", "%2C")
+
+			url = base + "?" + query
+		}
+	}
 
 	// err == nil indicates that the billing_project value was found
 	if bp, err := tpgresource.GetBillingProject(d, config); err == nil {
@@ -794,6 +861,32 @@ func flattenBigqueryAnalyticsHubListingBigqueryDatasetSelectedResourcesTable(v i
 	return v
 }
 
+func flattenBigqueryAnalyticsHubListingPubsubTopic(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
+	if v == nil {
+		return nil
+	}
+	original := v.(map[string]interface{})
+	if len(original) == 0 {
+		return nil
+	}
+	transformed := make(map[string]interface{})
+	transformed["topic"] =
+		flattenBigqueryAnalyticsHubListingPubsubTopicTopic(original["topic"], d, config)
+	transformed["data_affinity_regions"] =
+		flattenBigqueryAnalyticsHubListingPubsubTopicDataAffinityRegions(original["dataAffinityRegions"], d, config)
+	return []interface{}{transformed}
+}
+func flattenBigqueryAnalyticsHubListingPubsubTopicTopic(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
+	return v
+}
+
+func flattenBigqueryAnalyticsHubListingPubsubTopicDataAffinityRegions(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
+	if v == nil {
+		return v
+	}
+	return schema.NewSet(schema.HashString, v.([]interface{}))
+}
+
 func flattenBigqueryAnalyticsHubListingRestrictedExportConfig(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
 	if v == nil {
 		return nil
@@ -979,6 +1072,41 @@ func expandBigqueryAnalyticsHubListingBigqueryDatasetSelectedResourcesTable(v in
 	return v, nil
 }
 
+func expandBigqueryAnalyticsHubListingPubsubTopic(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
+	l := v.([]interface{})
+	if len(l) == 0 || l[0] == nil {
+		return nil, nil
+	}
+	raw := l[0]
+	original := raw.(map[string]interface{})
+	transformed := make(map[string]interface{})
+
+	transformedTopic, err := expandBigqueryAnalyticsHubListingPubsubTopicTopic(original["topic"], d, config)
+	if err != nil {
+		return nil, err
+	} else if val := reflect.ValueOf(transformedTopic); val.IsValid() && !tpgresource.IsEmptyValue(val) {
+		transformed["topic"] = transformedTopic
+	}
+
+	transformedDataAffinityRegions, err := expandBigqueryAnalyticsHubListingPubsubTopicDataAffinityRegions(original["data_affinity_regions"], d, config)
+	if err != nil {
+		return nil, err
+	} else if val := reflect.ValueOf(transformedDataAffinityRegions); val.IsValid() && !tpgresource.IsEmptyValue(val) {
+		transformed["dataAffinityRegions"] = transformedDataAffinityRegions
+	}
+
+	return transformed, nil
+}
+
+func expandBigqueryAnalyticsHubListingPubsubTopicTopic(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
+	return v, nil
+}
+
+func expandBigqueryAnalyticsHubListingPubsubTopicDataAffinityRegions(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
+	v = v.(*schema.Set).List()
+	return v, nil
+}
+
 func expandBigqueryAnalyticsHubListingRestrictedExportConfig(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
 	l := v.([]interface{})
 	if len(l) == 0 || l[0] == nil {

google-beta/services/bigqueryanalyticshub/resource_bigquery_analytics_hub_listing_generated_meta.yaml

@@ -25,6 +25,8 @@ fields:
   - field: 'primary_contact'
   - field: 'publisher.name'
   - field: 'publisher.primary_contact'
+  - field: 'pubsub_topic.data_affinity_regions'
+  - field: 'pubsub_topic.topic'
   - field: 'request_access'
   - field: 'restricted_export_config.enabled'
   - field: 'restricted_export_config.restrict_direct_table_access'

google-beta/services/bigqueryanalyticshub/resource_bigquery_analytics_hub_listing_generated_test.go

@@ -291,6 +291,62 @@ resource "google_bigquery_dataset" "listing_log_email" {
 `, context)
 }
 
+func TestAccBigqueryAnalyticsHubListing_bigqueryAnalyticshubListingPubsubExample(t *testing.T) {
+	t.Parallel()
+
+	context := map[string]interface{}{
+		"random_suffix": acctest.RandString(t, 10),
+	}
+
+	acctest.VcrTest(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+		CheckDestroy:             testAccCheckBigqueryAnalyticsHubListingDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccBigqueryAnalyticsHubListing_bigqueryAnalyticshubListingPubsubExample(context),
+			},
+			{
+				ResourceName:            "google_bigquery_analytics_hub_listing.listing",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"data_exchange_id", "listing_id", "location"},
+			},
+		},
+	})
+}
+
+func testAccBigqueryAnalyticsHubListing_bigqueryAnalyticshubListingPubsubExample(context map[string]interface{}) string {
+	return acctest.Nprintf(`
+resource "google_bigquery_analytics_hub_data_exchange" "listing" {
+  location         = "US"
+  data_exchange_id = "tf_test_tf_test_pubsub_data_exchange%{random_suffix}"
+  display_name     = "tf_test_tf_test_pubsub_data_exchange%{random_suffix}"
+  description      = "Example for pubsub topic source%{random_suffix}"
+}
+
+resource "google_pubsub_topic" "tf_test_pubsub_topic" { 
+  name    = "tf_test_test_pubsub%{random_suffix}" 
+}
+
+resource "google_bigquery_analytics_hub_listing" "listing" {
+  location         = "US"
+  data_exchange_id = google_bigquery_analytics_hub_data_exchange.listing.data_exchange_id
+  listing_id       = "tf_test_tf_test_pubsub_listing%{random_suffix}"
+  display_name     = "tf_test_tf_test_pubsub_listing%{random_suffix}"
+  description      = "Example for pubsub topic source%{random_suffix}"
+
+  pubsub_topic {
+    topic = google_pubsub_topic.tf_test_pubsub_topic.id
+    data_affinity_regions = [
+      "us-central1",
+      "europe-west1"
+    ]
+  }
+}
+`, context)
+}
+
 func testAccCheckBigqueryAnalyticsHubListingDestroyProducer(t *testing.T) func(s *terraform.State) error {
 	return func(s *terraform.State) error {
 		for name, rs := range s.RootModule().Resources {

google-beta/services/bigqueryanalyticshub/resource_bigquery_analytics_hub_listing_test.go

@@ -46,6 +46,26 @@ func TestAccBigqueryAnalyticsHubListing_bigqueryAnalyticshubListingUpdate(t *tes
 			{
 				Config: testAccBigqueryAnalyticsHubListing_bigqueryAnalyticshubListingUpdate(context),
 			},
+			{
+				ResourceName:      "google_bigquery_analytics_hub_listing.listing",
+				ImportState:       true,
+				ImportStateVerify: true,
+			},
+			{
+				Config: testAccBigqueryAnalyticsHubListing_pubsubListingUpdateConfig(context, `["us-central1"]`, "Example for pubsub topic source - initial"),
+				Check: resource.ComposeTestCheckFunc(
+					// Verify initial state for Pub/Sub listing
+					resource.TestCheckResourceAttr("google_bigquery_analytics_hub_listing.listing_pubsub", "pubsub_topic.0.data_affinity_regions.#", "1"),
+					resource.TestCheckResourceAttr("google_bigquery_analytics_hub_listing.listing_pubsub", "pubsub_topic.0.data_affinity_regions.0", "us-central1"),
+					resource.TestCheckResourceAttr("google_bigquery_analytics_hub_listing.listing_pubsub", "description", "Example for pubsub topic source - initial"),
+				),
+			},
+			// Step 7: Import the updated Pub/Sub Topic listing to verify import after update.
+			{
+				ResourceName:      "google_bigquery_analytics_hub_listing.listing_pubsub",
+				ImportState:       true,
+				ImportStateVerify: true,
+			},
 		},
 	})
 }
@@ -79,3 +99,45 @@ resource "google_bigquery_dataset" "listing" {
 }
 `, context)
 }
+func testAccBigqueryAnalyticsHubListing_pubsubListingUpdateConfig(context map[string]interface{}, dataAffinityRegionsHCL string, description string) string {
+	// Create a mutable copy of the context map
+	updatedContext := make(map[string]interface{})
+	for k, v := range context {
+		updatedContext[k] = v
+	}
+
+	// Directly assign the HCL string for data_affinity_regions and the description.
+	// dataAffinityRegionsHCL will be something like `["us-central1"]` or `["us-central1", "europe-west1"]`
+	updatedContext["data_affinity_regions_hcl"] = dataAffinityRegionsHCL
+	updatedContext["description_hcl"] = description
+
+	return acctest.Nprintf(`
+# Separate Data Exchange for the Pub/Sub listing to prevent conflicts
+resource "google_bigquery_analytics_hub_data_exchange" "listing_pubsub" {
+  location         = "US"
+  data_exchange_id = "tf_test_pubsub_data_exchange_update_%{random_suffix}"
+  display_name     = "tf_test_pubsub_data_exchange_update_%{random_suffix}"
+  description      = "Example for pubsub topic source - data exchange%{random_suffix}"
+}
+
+# Pub/Sub Topic used as the source for the listing
+resource "google_pubsub_topic" "tf_test_pubsub_topic" {
+  name = "tf_test_test_pubsub_update_%{random_suffix}"
+}
+
+# BigQuery Analytics Hub Listing sourced from the Pub/Sub Topic
+resource "google_bigquery_analytics_hub_listing" "listing_pubsub" {
+  location         = "US"
+  data_exchange_id = google_bigquery_analytics_hub_data_exchange.listing_pubsub.data_exchange_id
+  listing_id       = "tf_test_pubsub_listing_update_%{random_suffix}"
+  display_name     = "tf_test_pubsub_listing_update_%{random_suffix}"
+  description      = "%{description_hcl}" 
+  primary_contact  = "[email protected]" 
+
+  pubsub_topic {
+    topic               = google_pubsub_topic.tf_test_pubsub_topic.id
+    data_affinity_regions = %{data_affinity_regions_hcl} 
+  }
+}
+`, updatedContext)
+}