feat: (storage) added field aws_s3_compatible_data_source for google_storage_transfer_job resource (#14835)

Author: gurusai-voleti

mmv1/third_party/terraform/services/storagetransfer/resource_storage_transfer_job.go

@@ -83,6 +83,7 @@ var (
 		"transfer_spec.0.azure_blob_storage_data_source",
 		"transfer_spec.0.posix_data_source",
 		"transfer_spec.0.hdfs_data_source",
+		"transfer_spec.0.aws_s3_compatible_data_source",
 	}
 	transferSpecDataSinkKeys = []string{
 		"transfer_spec.0.gcs_data_sink",
@@ -293,6 +294,14 @@ func ResourceStorageTransferJob() *schema.Resource {
 							ExactlyOneOf: transferSpecDataSourceKeys,
 							Description:  `An HDFS Storage data source.`,
 						},
+						"aws_s3_compatible_data_source": {
+							Type:         schema.TypeList,
+							Optional:     true,
+							MaxItems:     1,
+							Elem:         awsS3CompatibleDataSchema(),
+							ExactlyOneOf: transferSpecDataSourceKeys,
+							Description:  `An AWS S3 Compatible data source.`,
+						},
 					},
 				},
 				Description: `Transfer specification.`,
@@ -887,6 +896,69 @@ func azureBlobStorageDataSchema() *schema.Resource {
 	}
 }
 
+func awsS3CompatibleDataSchema() *schema.Resource {
+	return &schema.Resource{
+		Schema: map[string]*schema.Schema{
+			"bucket_name": {
+				Type:        schema.TypeString,
+				Required:    true,
+				Description: `Name of the bucket.`,
+			},
+			"path": {
+				Type:        schema.TypeString,
+				Optional:    true,
+				Description: `Specifies the path to transfer objects.`,
+			},
+			"endpoint": {
+				Type:        schema.TypeString,
+				Required:    true,
+				Description: `Endpoint of the storage service.`,
+			},
+			"region": {
+				Type:        schema.TypeString,
+				Optional:    true,
+				Description: `Specifies the region to sign requests with. This can be left blank if requests should be signed with an empty region.`,
+			},
+			"s3_metadata": {
+				Type:        schema.TypeList,
+				Optional:    true,
+				MaxItems:    1,
+				Description: `S3 compatible metadata.`,
+				Elem: &schema.Resource{
+					Schema: map[string]*schema.Schema{
+						"auth_method": {
+							Type:         schema.TypeString,
+							Optional:     true,
+							ValidateFunc: verify.ValidateEnum([]string{"AUTH_METHOD_UNSPECIFIED", "AUTH_METHOD_AWS_SIGNATURE_V4", "AUTH_METHOD_AWS_SIGNATURE_V2"}),
+							Description:  `Authentication and authorization method used by the storage service. When not specified, Transfer Service will attempt to determine right auth method to use.`,
+						},
+						"request_model": {
+							Type:         schema.TypeString,
+							Optional:     true,
+							Default:      "REQUEST_MODEL_VIRTUAL_HOSTED_STYLE",
+							ValidateFunc: verify.ValidateEnum([]string{"REQUEST_MODEL_VIRTUAL_HOSTED_STYLE", "REQUEST_MODEL_UNSPECIFIED", "REQUEST_MODEL_PATH_STYLE"}),
+							Description:  `API request model used to call the storage service. When not specified, the default value of RequestModel REQUEST_MODEL_VIRTUAL_HOSTED_STYLE is used.`,
+						},
+						"protocol": {
+							Type:         schema.TypeString,
+							Optional:     true,
+							Default:      "NETWORK_PROTOCOL_HTTPS",
+							ValidateFunc: verify.ValidateEnum([]string{"NETWORK_PROTOCOL_UNSPECIFIED", "NETWORK_PROTOCOL_HTTPS", "NETWORK_PROTOCOL_HTTP"}),
+							Description:  `The network protocol of the agent. When not specified, the default value of NetworkProtocol NETWORK_PROTOCOL_HTTPS is used.`,
+						},
+						"list_api": {
+							Type:         schema.TypeString,
+							Optional:     true,
+							ValidateFunc: verify.ValidateEnum([]string{"LIST_API_UNSPECIFIED", "LIST_OBJECTS_V2", "LIST_OBJECTS"}),
+							Description:  `The Listing API to use for discovering objects. When not specified, Transfer Service will attempt to determine the right API to use.`,
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
 func diffSuppressEmptyStartTimeOfDay(k, old, new string, d *schema.ResourceData) bool {
 	return k == "schedule.0.start_time_of_day.#" && old == "1" && new == "0"
 }
@@ -1501,6 +1573,73 @@ func flattenHdfsData(hdfsData *storagetransfer.HdfsData) []map[string]interface{
 	return []map[string]interface{}{data}
 }
 
+func expandAwsS3CompatibleData(awsS3CompatibleDataSchema []interface{}) *storagetransfer.AwsS3CompatibleData {
+	if len(awsS3CompatibleDataSchema) == 0 || awsS3CompatibleDataSchema[0] == nil {
+		return nil
+	}
+
+	awsS3CompatibleData := awsS3CompatibleDataSchema[0].(map[string]interface{})
+	result := &storagetransfer.AwsS3CompatibleData{
+		BucketName: awsS3CompatibleData["bucket_name"].(string),
+		Path:       awsS3CompatibleData["path"].(string),
+		Endpoint:   awsS3CompatibleData["endpoint"].(string),
+		Region:     awsS3CompatibleData["region"].(string),
+	}
+
+	if v, ok := awsS3CompatibleData["s3_metadata"].([]interface{}); ok {
+		result.S3Metadata = expandS3Metadata(v)
+	}
+	return result
+}
+
+func expandS3Metadata(s3Metadata []interface{}) *storagetransfer.S3CompatibleMetadata {
+	if len(s3Metadata) == 0 || s3Metadata[0] == nil {
+		return nil
+	}
+	metadata := s3Metadata[0].(map[string]interface{})
+	data := &storagetransfer.S3CompatibleMetadata{
+		AuthMethod:   metadata["auth_method"].(string),
+		ListApi:      metadata["list_api"].(string),
+		RequestModel: metadata["request_model"].(string),
+		Protocol:     metadata["protocol"].(string),
+	}
+	return data
+}
+
+func flattenAwsS3CompatibleData(awsS3CompatibleData *storagetransfer.AwsS3CompatibleData, d *schema.ResourceData) []map[string]interface{} {
+	data := map[string]interface{}{
+		"bucket_name": awsS3CompatibleData.BucketName,
+		"path":        awsS3CompatibleData.Path,
+		"endpoint":    awsS3CompatibleData.Endpoint,
+		"region":      awsS3CompatibleData.Region,
+	}
+	if awsS3CompatibleData.S3Metadata != nil {
+		data["s3_metadata"] = flattenS3MetaData(awsS3CompatibleData.S3Metadata, d)
+	}
+
+	return []map[string]interface{}{data}
+}
+
+func flattenS3MetaData(s3MetaData *storagetransfer.S3CompatibleMetadata, d *schema.ResourceData) []map[string]interface{} {
+	s3Metadata := map[string]interface{}{
+		"protocol":      s3MetaData.Protocol,
+		"request_model": s3MetaData.RequestModel,
+	}
+	if d.Get("transfer_spec.0.aws_s3_compatible_data_source.0.s3_metadata.0.auth_method") == "AUTH_METHOD_UNSPECIFIED" {
+		s3Metadata["auth_method"] = d.Get("transfer_spec.0.aws_s3_compatible_data_source.0.s3_metadata.0.auth_method")
+	} else {
+		s3Metadata["auth_method"] = s3MetaData.AuthMethod
+	}
+
+	if d.Get("transfer_spec.0.aws_s3_compatible_data_source.0.s3_metadata.0.list_api") == "LIST_API_UNSPECIFIED" {
+		s3Metadata["list_api"] = d.Get("transfer_spec.0.aws_s3_compatible_data_source.0.s3_metadata.0.list_api")
+	} else {
+		s3Metadata["list_api"] = s3MetaData.ListApi
+	}
+
+	return []map[string]interface{}{s3Metadata}
+}
+
 func expandAzureCredentials(azureCredentials []interface{}) *storagetransfer.AzureCredentials {
 	if len(azureCredentials) == 0 || azureCredentials[0] == nil {
 		return nil
@@ -1665,6 +1804,7 @@ func expandTransferSpecs(transferSpecs []interface{}) *storagetransfer.TransferS
 		AzureBlobStorageDataSource: expandAzureBlobStorageData(transferSpec["azure_blob_storage_data_source"].([]interface{})),
 		PosixDataSource:            expandPosixData(transferSpec["posix_data_source"].([]interface{})),
 		HdfsDataSource:             expandHdfsData(transferSpec["hdfs_data_source"].([]interface{})),
+		AwsS3CompatibleDataSource:  expandAwsS3CompatibleData(transferSpec["aws_s3_compatible_data_source"].([]interface{})),
 	}
 }
 
@@ -1705,6 +1845,8 @@ func flattenTransferSpec(transferSpec *storagetransfer.TransferSpec, d *schema.R
 		data["posix_data_source"] = flattenPosixData(transferSpec.PosixDataSource)
 	} else if transferSpec.HdfsDataSource != nil {
 		data["hdfs_data_source"] = flattenHdfsData(transferSpec.HdfsDataSource)
+	} else if transferSpec.AwsS3CompatibleDataSource != nil {
+		data["aws_s3_compatible_data_source"] = flattenAwsS3CompatibleData(transferSpec.AwsS3CompatibleDataSource, d)
 	}
 
 	return []map[string]interface{}{data}

mmv1/third_party/terraform/services/storagetransfer/resource_storage_transfer_job_meta.yaml

@@ -57,6 +57,14 @@ fields:
   - field: 'transfer_spec.azure_blob_storage_data_source.federated_identity_config.tenant_id'
   - field: 'transfer_spec.azure_blob_storage_data_source.path'
   - field: 'transfer_spec.azure_blob_storage_data_source.storage_account'
+  - field: 'transfer_spec.aws_s3_compatible_data_source.bucket_name'
+  - field: 'transfer_spec.aws_s3_compatible_data_source.path'
+  - field: 'transfer_spec.aws_s3_compatible_data_source.region'
+  - field: 'transfer_spec.aws_s3_compatible_data_source.endpoint'
+  - field: 'transfer_spec.aws_s3_compatible_data_source.s3_metadata.request_model'
+  - field: 'transfer_spec.aws_s3_compatible_data_source.s3_metadata.protocol'
+  - field: 'transfer_spec.aws_s3_compatible_data_source.s3_metadata.list_api'
+  - field: 'transfer_spec.aws_s3_compatible_data_source.s3_metadata.auth_method'
   - field: 'transfer_spec.gcs_data_sink.bucket_name'
   - field: 'transfer_spec.gcs_data_sink.path'
   - field: 'transfer_spec.gcs_data_source.bucket_name'