add support to reference cross-project agent pool in storagetransfer (#7306) (#5262)

Signed-off-by: Modular Magician <[email protected]>
Co-authored-by: Anuhya P <[email protected]>

Author: modular-magician

.changelog/7306.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+storagetransfer: added general field `sink_agent_pool_name` and `source_agent_pool_name` to `google_storage_transfer_job`
+```

google-beta/resource_storage_transfer_job.go

@@ -82,6 +82,20 @@ func ResourceStorageTransferJob() *schema.Resource {
 					Schema: map[string]*schema.Schema{
 						"object_conditions": objectConditionsSchema(),
 						"transfer_options":  transferOptionsSchema(),
+						"source_agent_pool_name": {
+							Type:        schema.TypeString,
+							Optional:    true,
+							Computed:    true,
+							ForceNew:    true,
+							Description: `Specifies the agent pool name associated with the posix data source. When unspecified, the default name is used.`,
+						},
+						"sink_agent_pool_name": {
+							Type:        schema.TypeString,
+							Optional:    true,
+							Computed:    true,
+							ForceNew:    true,
+							Description: `Specifies the agent pool name associated with the posix data source. When unspecified, the default name is used.`,
+						},
 						"gcs_data_sink": {
 							Type:         schema.TypeList,
 							Optional:     true,
@@ -1064,6 +1078,8 @@ func expandTransferSpecs(transferSpecs []interface{}) *storagetransfer.TransferS
 
 	transferSpec := transferSpecs[0].(map[string]interface{})
 	return &storagetransfer.TransferSpec{
+		SourceAgentPoolName:        transferSpec["source_agent_pool_name"].(string),
+		SinkAgentPoolName:          transferSpec["sink_agent_pool_name"].(string),
 		GcsDataSink:                expandGcsData(transferSpec["gcs_data_sink"].([]interface{})),
 		PosixDataSink:              expandPosixData(transferSpec["posix_data_sink"].([]interface{})),
 		ObjectConditions:           expandObjectConditions(transferSpec["object_conditions"].([]interface{})),
@@ -1076,8 +1092,13 @@ func expandTransferSpecs(transferSpecs []interface{}) *storagetransfer.TransferS
 	}
 }
 
-func flattenTransferSpec(transferSpec *storagetransfer.TransferSpec, d *schema.ResourceData) []map[string][]map[string]interface{} {
-	data := map[string][]map[string]interface{}{}
+func flattenTransferSpec(transferSpec *storagetransfer.TransferSpec, d *schema.ResourceData) []map[string]interface{} {
+
+	data := map[string]interface{}{}
+
+	data["sink_agent_pool_name"] = transferSpec.SinkAgentPoolName
+	data["source_agent_pool_name"] = transferSpec.SourceAgentPoolName
+
 	if transferSpec.GcsDataSink != nil {
 		data["gcs_data_sink"] = flattenGcsData(transferSpec.GcsDataSink)
 	}
@@ -1105,7 +1126,7 @@ func flattenTransferSpec(transferSpec *storagetransfer.TransferSpec, d *schema.R
 		data["posix_data_source"] = flattenPosixData(transferSpec.PosixDataSource)
 	}
 
-	return []map[string][]map[string]interface{}{data}
+	return []map[string]interface{}{data}
 }
 
 func usingPosix(transferSpec *storagetransfer.TransferSpec) bool {

google-beta/resource_storage_transfer_job_test.go

@@ -105,14 +105,15 @@ func TestAccStorageTransferJob_posixSource(t *testing.T) {
 
 	testDataSinkName := randString(t, 10)
 	testTransferJobDescription := randString(t, 10)
+	testSourceAgentPoolName := fmt.Sprintf("tf-test-source-agent-pool-%s", randString(t, 10))
 
 	vcrTest(t, resource.TestCase{
 		PreCheck:     func() { testAccPreCheck(t) },
 		Providers:    testAccProviders,
 		CheckDestroy: testAccStorageTransferJobDestroyProducer(t),
 		Steps: []resource.TestStep{
 			{
-				Config: testAccStorageTransferJob_posixSource(getTestProjectFromEnv(), testDataSinkName, testTransferJobDescription),
+				Config: testAccStorageTransferJob_posixSource(getTestProjectFromEnv(), testDataSinkName, testTransferJobDescription, testSourceAgentPoolName),
 			},
 			{
 				ResourceName:      "google_storage_transfer_job.transfer_job",
@@ -122,20 +123,20 @@ func TestAccStorageTransferJob_posixSource(t *testing.T) {
 		},
 	})
 }
-
 func TestAccStorageTransferJob_posixSink(t *testing.T) {
 	t.Parallel()
 
 	testDataSourceName := randString(t, 10)
 	testTransferJobDescription := randString(t, 10)
+	testSinkAgentPoolName := fmt.Sprintf("tf-test-sink-agent-pool-%s", randString(t, 10))
 
 	vcrTest(t, resource.TestCase{
 		PreCheck:     func() { testAccPreCheck(t) },
 		Providers:    testAccProviders,
 		CheckDestroy: testAccStorageTransferJobDestroyProducer(t),
 		Steps: []resource.TestStep{
 			{
-				Config: testAccStorageTransferJob_posixSink(getTestProjectFromEnv(), testDataSourceName, testTransferJobDescription),
+				Config: testAccStorageTransferJob_posixSink(getTestProjectFromEnv(), testDataSourceName, testTransferJobDescription, testSinkAgentPoolName),
 			},
 			{
 				ResourceName:      "google_storage_transfer_job.transfer_job",
@@ -580,7 +581,7 @@ resource "google_storage_transfer_job" "transfer_job" {
 `, project, dataSourceBucketName, project, dataSinkBucketName, project, transferJobDescription, project)
 }
 
-func testAccStorageTransferJob_posixSource(project string, dataSinkBucketName string, transferJobDescription string) string {
+func testAccStorageTransferJob_posixSource(project string, dataSinkBucketName string, transferJobDescription string, sourceAgentPoolName string) string {
 	return fmt.Sprintf(`
 data "google_storage_transfer_project_service_account" "default" {
   project = "%s"
@@ -605,11 +606,21 @@ resource "google_project_iam_member" "pubsub" {
   member  = "serviceAccount:${data.google_storage_transfer_project_service_account.default.email}"
 }
 
+resource "google_storage_transfer_agent_pool" "foo" {
+  name         = "%s"
+  bandwidth_limit {
+    limit_mbps = "120"
+  }
+
+  depends_on = [google_project_iam_member.pubsub]
+}
+
 resource "google_storage_transfer_job" "transfer_job" {
   description = "%s"
   project     = "%s"
 
   transfer_spec {
+    source_agent_pool_name = google_storage_transfer_agent_pool.foo.id
     posix_data_source {
     	root_directory = "/some/path"
     }
@@ -643,10 +654,10 @@ resource "google_storage_transfer_job" "transfer_job" {
     google_project_iam_member.pubsub
   ]
 }
-`, project, dataSinkBucketName, project, transferJobDescription, project)
+`, project, dataSinkBucketName, project, transferJobDescription, sourceAgentPoolName, project)
 }
 
-func testAccStorageTransferJob_posixSink(project string, dataSourceBucketName string, transferJobDescription string) string {
+func testAccStorageTransferJob_posixSink(project string, dataSourceBucketName string, transferJobDescription string, sinkAgentPoolName string) string {
 	return fmt.Sprintf(`
 data "google_storage_transfer_project_service_account" "default" {
   project = "%s"
@@ -671,11 +682,21 @@ resource "google_project_iam_member" "pubsub" {
   member  = "serviceAccount:${data.google_storage_transfer_project_service_account.default.email}"
 }
 
+resource "google_storage_transfer_agent_pool" "foo" {
+  name         = "%s"
+  bandwidth_limit {
+    limit_mbps = "120"
+  }
+
+  depends_on = [google_project_iam_member.pubsub]
+}
+
 resource "google_storage_transfer_job" "transfer_job" {
   description = "%s"
   project     = "%s"
 
   transfer_spec {
+    sink_agent_pool_name  = google_storage_transfer_agent_pool.foo.id
     posix_data_sink {
     	root_directory = "/some/path"
     }
@@ -708,7 +729,7 @@ resource "google_storage_transfer_job" "transfer_job" {
     google_project_iam_member.pubsub
   ]
 }
-`, project, dataSourceBucketName, project, transferJobDescription, project)
+`, project, dataSourceBucketName, project, sinkAgentPoolName, transferJobDescription, project)
 }
 
 func testAccStorageTransferJob_transferOptions(project string, dataSourceBucketName string, dataSinkBucketName string, transferJobDescription string, overwriteObjectsAlreadyExistingInSink bool, deleteObjectsUniqueInSink bool, deleteObjectsFromSourceAfterTransfer bool, overwriteWhenVal string, pubSubTopicName string) string {

website/docs/r/storage_transfer_job.html.markdown

@@ -129,6 +129,10 @@ The following arguments are supported:
 
 <a name="nested_transfer_spec"></a>The `transfer_spec` block supports:
 
+* `source_agent_pool_name` - (Optional) Specifies the agent pool name associated with the posix data source. When unspecified, the default name is used.
+
+* `sink_agent_pool_name` - (Optional) Specifies the agent pool name associated with the posix data sink. When unspecified, the default name is used.
+
 * `gcs_data_sink` - (Optional) A Google Cloud Storage data sink. Structure [documented below](#nested_gcs_data_sink).
 
 * `posix_data_sink` - (Optional) A POSIX data sink. Structure [documented below](#nested_posix_data_sink).