Added support for TransferManifest option in TransferSpec for google_storage_transfer_job (#15367)

Author: malhotrasagar2212

mmv1/third_party/terraform/services/storagetransfer/resource_storage_transfer_job.go

@@ -302,6 +302,13 @@ func ResourceStorageTransferJob() *schema.Resource {
 							ExactlyOneOf: transferSpecDataSourceKeys,
 							Description:  `An AWS S3 Compatible data source.`,
 						},
+						"transfer_manifest": {
+							Type:        schema.TypeList,
+							Optional:    true,
+							MaxItems:    1,
+							Elem:        transferManifest(),
+							Description: `A manifest file listing specific objects to transfer.`,
+						},
 					},
 				},
 				Description: `Transfer specification.`,
@@ -827,6 +834,19 @@ func hdfsDataSchema() *schema.Resource {
 	}
 }
 
+func transferManifest() *schema.Resource {
+	return &schema.Resource{
+		Schema: map[string]*schema.Schema{
+			"location": {
+				Type:         schema.TypeString,
+				Required:     true,
+				Description:  `Cloud Storage path to the manifest CSV.`,
+				ValidateFunc: validation.StringMatch(regexp.MustCompile(`^gs://[^/]+/.+`), "must be a Cloud path like gs://BUCKET/path/manifest.csv"),
+			},
+		},
+	}
+}
+
 func azureBlobStorageDataSchema() *schema.Resource {
 	return &schema.Resource{
 		Schema: map[string]*schema.Schema{
@@ -1617,6 +1637,26 @@ func flattenHdfsData(hdfsData *storagetransfer.HdfsData) []map[string]interface{
 	return []map[string]interface{}{data}
 }
 
+func expandTransferManifest(manifest []interface{}) *storagetransfer.TransferManifest {
+	if len(manifest) == 0 || manifest[0] == nil {
+		return nil
+	}
+
+	manifestFile := manifest[0].(map[string]interface{})
+	return &storagetransfer.TransferManifest{
+		Location: manifestFile["location"].(string),
+	}
+}
+
+func flattenTransferManifest(manifest *storagetransfer.TransferManifest) []map[string]interface{} {
+
+	return []map[string]interface{}{
+		{
+			"location": manifest.Location,
+		},
+	}
+}
+
 func expandAwsS3CompatibleData(awsS3CompatibleDataSchema []interface{}) *storagetransfer.AwsS3CompatibleData {
 	if len(awsS3CompatibleDataSchema) == 0 || awsS3CompatibleDataSchema[0] == nil {
 		return nil
@@ -1849,6 +1889,7 @@ func expandTransferSpecs(transferSpecs []interface{}) *storagetransfer.TransferS
 		PosixDataSource:            expandPosixData(transferSpec["posix_data_source"].([]interface{})),
 		HdfsDataSource:             expandHdfsData(transferSpec["hdfs_data_source"].([]interface{})),
 		AwsS3CompatibleDataSource:  expandAwsS3CompatibleData(transferSpec["aws_s3_compatible_data_source"].([]interface{})),
+		TransferManifest:           expandTransferManifest(transferSpec["transfer_manifest"].([]interface{})),
 	}
 }
 
@@ -1892,6 +1933,9 @@ func flattenTransferSpec(transferSpec *storagetransfer.TransferSpec, d *schema.R
 	} else if transferSpec.AwsS3CompatibleDataSource != nil {
 		data["aws_s3_compatible_data_source"] = flattenAwsS3CompatibleData(transferSpec.AwsS3CompatibleDataSource, d)
 	}
+	if transferSpec.TransferManifest != nil && transferSpec.TransferManifest.Location != "" {
+		data["transfer_manifest"] = flattenTransferManifest(transferSpec.TransferManifest)
+	}
 
 	return []map[string]interface{}{data}
 }

mmv1/third_party/terraform/services/storagetransfer/resource_storage_transfer_job_meta.yaml

@@ -81,6 +81,7 @@ fields:
   - field: 'transfer_spec.posix_data_source.root_directory'
   - field: 'transfer_spec.sink_agent_pool_name'
   - field: 'transfer_spec.source_agent_pool_name'
+  - field: 'transfer_spec.transfer_manifest.location'
   - field: 'transfer_spec.transfer_options.delete_objects_from_source_after_transfer'
   - field: 'transfer_spec.transfer_options.delete_objects_unique_in_sink'
   - field: 'transfer_spec.transfer_options.overwrite_objects_already_existing_in_sink'

mmv1/third_party/terraform/services/storagetransfer/resource_storage_transfer_job_test.go

@@ -664,6 +664,70 @@ func TestAccStorageTransferJob_awsS3CompatibleDataSource(t *testing.T) {
 	})
 }
 
+func TestAccStorageTransferJob_transferManifest(t *testing.T) {
+	t.Parallel()
+
+	project := envvar.GetTestProjectFromEnv()
+	src := acctest.RandString(t, 10)
+	dst := acctest.RandString(t, 10)
+	desc := acctest.RandString(t, 10)
+
+	manifestV1 := fmt.Sprintf("manifest-%s-v1.csv", acctest.RandString(t, 6))
+	manifestV2 := fmt.Sprintf("manifest-%s-v2.csv", acctest.RandString(t, 6))
+
+	acctest.VcrTest(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+		CheckDestroy:             testAccStorageTransferJobDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccStorageTransferJob_withTransferManifest(project, src, dst, desc, manifestV1),
+				Check: resource.ComposeAggregateTestCheckFunc(
+					resource.TestCheckResourceAttr(
+						"google_storage_transfer_job.transfer_job",
+						"transfer_spec.0.transfer_manifest.0.location",
+						fmt.Sprintf("gs://%s/%s", src, manifestV1),
+					),
+				),
+			},
+			{
+				ResourceName:      "google_storage_transfer_job.transfer_job",
+				ImportState:       true,
+				ImportStateVerify: true,
+			},
+			{
+				Config: testAccStorageTransferJob_withTransferManifest(project, src, dst, desc, manifestV2),
+				Check: resource.ComposeAggregateTestCheckFunc(
+					resource.TestCheckResourceAttr(
+						"google_storage_transfer_job.transfer_job",
+						"transfer_spec.0.transfer_manifest.0.location",
+						fmt.Sprintf("gs://%s/%s", src, manifestV2),
+					),
+				),
+			},
+			{
+				ResourceName:      "google_storage_transfer_job.transfer_job",
+				ImportState:       true,
+				ImportStateVerify: true,
+			},
+			{
+				Config: testAccStorageTransferJob_withoutTransferManifest(project, src, dst, desc),
+				Check: resource.ComposeAggregateTestCheckFunc(
+					resource.TestCheckNoResourceAttr(
+						"google_storage_transfer_job.transfer_job",
+						"transfer_spec.0.transfer_manifest.0.location",
+					),
+				),
+			},
+			{
+				ResourceName:      "google_storage_transfer_job.transfer_job",
+				ImportState:       true,
+				ImportStateVerify: true,
+			},
+		},
+	})
+}
+
 func testAccStorageTransferJobDestroyProducer(t *testing.T) func(s *terraform.State) error {
 	return func(s *terraform.State) error {
 		config := acctest.GoogleProviderConfig(t)
@@ -2886,3 +2950,155 @@ resource "google_storage_transfer_job" "with_sa" {
 }
 `, project, dataSourceBucketName, project, dataSinkBucketName, project, description, project)
 }
+
+func testAccStorageTransferJob_withTransferManifest(project, dataSourceBucketName, dataSinkBucketName, transferJobDescription, manifestObjectName string) string {
+	return fmt.Sprintf(`
+data "google_storage_transfer_project_service_account" "default" {
+  project = "%[1]s"
+}
+
+resource "google_storage_bucket" "data_source" {
+  project                      = "%[1]s"
+  name                         = "%[2]s"
+  location                     = "US"
+  force_destroy                = true
+  uniform_bucket_level_access = true
+}
+
+resource "google_storage_bucket" "data_sink" {
+  project                      = "%[1]s"
+  name                         = "%[3]s"
+  location                     = "US"
+  force_destroy                = true
+  uniform_bucket_level_access = true
+}
+
+resource "google_storage_bucket_iam_member" "source_iam" {
+  bucket = google_storage_bucket.data_source.name
+  role   = "roles/storage.admin"
+  member = "serviceAccount:${data.google_storage_transfer_project_service_account.default.email}"
+}
+
+resource "google_storage_bucket_iam_member" "sink_iam" {
+  bucket = google_storage_bucket.data_sink.name
+  role   = "roles/storage.admin"
+  member = "serviceAccount:${data.google_storage_transfer_project_service_account.default.email}"
+}
+
+resource "google_storage_bucket_object" "manifest"{
+  name    = "%[5]s"
+  bucket  = google_storage_bucket.data_source.name
+  content = "dummy-manifest-content"
+  depends_on = [
+    google_storage_bucket_iam_member.sink_iam, google_storage_bucket_iam_member.source_iam,
+  ]
+}
+
+resource "google_storage_transfer_job" "transfer_job" {
+  description = "%[4]s"
+  project     = "%[1]s"
+
+  transfer_spec {
+    transfer_manifest {
+      location = "gs://${google_storage_bucket.data_source.name}/${google_storage_bucket_object.manifest.name}"
+    }
+    gcs_data_source {
+      bucket_name = google_storage_bucket.data_source.name
+      path        = "foo/bar/"
+    }
+    gcs_data_sink {
+      bucket_name = google_storage_bucket.data_sink.name
+      path        = "foo/bar/"
+    }
+  }
+
+  schedule {
+    schedule_start_date {
+      year  = 2023
+      month = 1
+      day   = 13
+    }
+    schedule_end_date {
+      year  = 2023
+      month = 1
+      day   = 13
+    }
+  }
+
+  depends_on = [
+    google_storage_bucket_object.manifest,
+  ]
+
+}
+`, project, dataSourceBucketName, dataSinkBucketName, transferJobDescription, manifestObjectName)
+}
+
+func testAccStorageTransferJob_withoutTransferManifest(project, dataSourceBucketName, dataSinkBucketName, transferJobDescription string) string {
+	return fmt.Sprintf(`
+data "google_storage_transfer_project_service_account" "default" {
+  project = "%[1]s"
+}
+
+resource "google_storage_bucket" "data_source" {
+  project                      = "%[1]s"
+  name                         = "%[2]s"
+  location                     = "US"
+  force_destroy                = true
+  uniform_bucket_level_access = true
+}
+
+resource "google_storage_bucket" "data_sink" {
+  project                      = "%[1]s"
+  name                         = "%[3]s"
+  location                     = "US"
+  force_destroy                = true
+  uniform_bucket_level_access = true
+}
+
+resource "google_storage_bucket_iam_member" "source_iam" {
+  bucket = google_storage_bucket.data_source.name
+  role   = "roles/storage.admin"
+  member = "serviceAccount:${data.google_storage_transfer_project_service_account.default.email}"
+}
+
+resource "google_storage_bucket_iam_member" "sink_iam" {
+  bucket = google_storage_bucket.data_sink.name
+  role   = "roles/storage.admin"
+  member = "serviceAccount:${data.google_storage_transfer_project_service_account.default.email}"
+}
+
+resource "google_storage_transfer_job" "transfer_job" {
+  description = "%[4]s"
+  project     = "%[1]s"
+
+  transfer_spec {
+    gcs_data_source {
+      bucket_name = google_storage_bucket.data_source.name
+      path        = "foo/bar/"
+    }
+    gcs_data_sink {
+      bucket_name = google_storage_bucket.data_sink.name
+      path        = "foo/bar/"
+    }
+  }
+
+  schedule {
+    schedule_start_date {
+      year  = 2023
+      month = 1
+      day   = 13
+    }
+    schedule_end_date {
+      year  = 2023
+      month = 1
+      day   = 13
+    }
+  }
+
+  depends_on = [
+    google_storage_bucket_iam_member.sink_iam, google_storage_bucket_iam_member.source_iam,
+  ]
+
+}
+`, project, dataSourceBucketName, dataSinkBucketName, transferJobDescription)
+}

mmv1/third_party/terraform/website/docs/r/storage_transfer_job.html.markdown

@@ -176,6 +176,8 @@ The following arguments are supported:
 
 * `aws_s3_compatible_data_source` - (Optional) An AWS S3 Compatible data source. Structure [documented below](#nested_aws_s3_compatible_data_source).
 
+* `transfer_manifest` - (Optional) Use a manifest file to limit which object are transferred. See [Storage Transfer Service manifest file format](https://cloud.google.com/storage-transfer/docs/manifest). Structure [documented below](#nested_transfer_manifest).
+
 <a name="nested_replication_spec"></a>The `replication_spec` block supports:
 
 * `gcs_data_sink` - (Optional) A Google Cloud Storage data sink. Structure [documented below](#nested_gcs_data_sink).
@@ -293,6 +295,10 @@ A duration in seconds with up to nine fractional digits, terminated by 's'. Exam
 
 * `credentials_secret` - (Optional)  The Resource name of a secret in Secret Manager. AWS credentials must be stored in Secret Manager in JSON format. If credentials_secret is specified, do not specify role_arn or aws_access_key. Format: `projects/{projectNumber}/secrets/{secret_name}`.
 
+<a name="nested_transfer_manifest"></a>The `transfer_manifest` block supports:
+
+* `location` - (Required) The **GCS URI** to the manifest file (CSV or line-delimited). Example: `gs://my-bucket/manifest.csv`
+
 The `aws_access_key` block supports:
 
 * `access_key_id` - (Required) AWS Key ID.