add sensitive_params to bigquery_data_transfer_config (#3937) (#2451)

* suppress diff for secret_access_key on bigquery data transfer params

* add sensitiveParams for secret access key

* add customize diff, fix spelling

* add custom import and post create

Signed-off-by: Modular Magician <[email protected]>

Author: modular-magician

.changelog/3937.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+bigquerydatatransfer: fixed `params.secret_access_key` perma-diff for AWS S3 data transfer config types by adding a `sensitive_params` block with the `secret_access_key` attribute.
+```

google-beta/resource_bigquery_data_transfer_config.go

@@ -25,6 +25,19 @@ import (
 	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
 )
 
+var sensitiveParams = []string{"secret_access_key"}
+
+func sensitiveParamCustomizeDiff(diff *schema.ResourceDiff, v interface{}) error {
+	for _, sp := range sensitiveParams {
+		mapLabel := diff.Get("params." + sp).(string)
+		authLabel := diff.Get("sensitive_params.0." + sp).(string)
+		if mapLabel != "" && authLabel != "" {
+			return fmt.Errorf("Sensitive param [%s] cannot be set in both `params` and the `sensitive_params` block.", sp)
+		}
+	}
+	return nil
+}
+
 func resourceBigqueryDataTransferConfig() *schema.Resource {
 	return &schema.Resource{
 		Create: resourceBigqueryDataTransferConfigCreate,
@@ -42,6 +55,8 @@ func resourceBigqueryDataTransferConfig() *schema.Resource {
 			Delete: schema.DefaultTimeout(4 * time.Minute),
 		},
 
+		CustomizeDiff: sensitiveParamCustomizeDiff,
+
 		Schema: map[string]*schema.Schema{
 			"data_source_id": {
 				Type:        schema.TypeString,
@@ -106,6 +121,28 @@ about the format here:
 https://cloud.google.com/appengine/docs/flexible/python/scheduling-jobs-with-cron-yaml#the_schedule_format
 NOTE: the granularity should be at least 8 hours, or less frequent.`,
 			},
+			"sensitive_params": {
+				Type:     schema.TypeList,
+				Optional: true,
+				Description: `Different parameters are configured primarily using the the 'params' field on this
+resource. This block contains the parameters which contain secrets or passwords so that they can be marked
+sensitive and hidden from plan output. The name of the field, eg: secret_access_key, will be the key
+in the 'params' map in the api request.
+
+Credentials may not be specified in both locations and will cause an error. Changing from one location
+to a different credential configuration in the config will require an apply to update state.`,
+				MaxItems: 1,
+				Elem: &schema.Resource{
+					Schema: map[string]*schema.Schema{
+						"secret_access_key": {
+							Type:        schema.TypeString,
+							Required:    true,
+							Description: `The Secret Access Key of the AWS account transferring data from.`,
+							Sensitive:   true,
+						},
+					},
+				},
+			},
 			"service_account_name": {
 				Type:     schema.TypeString,
 				Optional: true,
@@ -186,6 +223,11 @@ func resourceBigqueryDataTransferConfigCreate(d *schema.ResourceData, meta inter
 		obj["params"] = paramsProp
 	}
 
+	obj, err = resourceBigqueryDataTransferConfigEncoder(d, meta, obj)
+	if err != nil {
+		return err
+	}
+
 	url, err := replaceVars(d, config, "{{BigqueryDataTransferBasePath}}projects/{{project}}/locations/{{location}}/transferConfigs?serviceAccountName={{service_account_name}}")
 	if err != nil {
 		return err
@@ -267,6 +309,18 @@ func resourceBigqueryDataTransferConfigRead(d *schema.ResourceData, meta interfa
 		return handleNotFoundError(err, d, fmt.Sprintf("BigqueryDataTransferConfig %q", d.Id()))
 	}
 
+	res, err = resourceBigqueryDataTransferConfigDecoder(d, meta, res)
+	if err != nil {
+		return err
+	}
+
+	if res == nil {
+		// Decoding the object has resulted in it being gone. It may be marked deleted
+		log.Printf("[DEBUG] Removing BigqueryDataTransferConfig because it no longer exists.")
+		d.SetId("")
+		return nil
+	}
+
 	if err := d.Set("project", project); err != nil {
 		return fmt.Errorf("Error reading Config: %s", err)
 	}
@@ -351,6 +405,11 @@ func resourceBigqueryDataTransferConfigUpdate(d *schema.ResourceData, meta inter
 		obj["params"] = paramsProp
 	}
 
+	obj, err = resourceBigqueryDataTransferConfigEncoder(d, meta, obj)
+	if err != nil {
+		return err
+	}
+
 	url, err := replaceVars(d, config, "{{BigqueryDataTransferBasePath}}{{name}}")
 	if err != nil {
 		return err
@@ -547,3 +606,40 @@ func expandBigqueryDataTransferConfigParams(v interface{}, d TerraformResourceDa
 	}
 	return m, nil
 }
+
+func resourceBigqueryDataTransferConfigEncoder(d *schema.ResourceData, meta interface{}, obj map[string]interface{}) (map[string]interface{}, error) {
+	paramMap, ok := obj["params"]
+	if !ok {
+		paramMap = make(map[string]string)
+	}
+
+	var params map[string]string
+	params = paramMap.(map[string]string)
+
+	for _, sp := range sensitiveParams {
+		if auth, _ := d.GetOkExists("sensitive_params.0." + sp); auth != "" {
+			params[sp] = auth.(string)
+		}
+	}
+
+	obj["params"] = params
+
+	return obj, nil
+}
+
+func resourceBigqueryDataTransferConfigDecoder(d *schema.ResourceData, meta interface{}, res map[string]interface{}) (map[string]interface{}, error) {
+	if paramMap, ok := res["params"]; ok {
+		params := paramMap.(map[string]interface{})
+		for _, sp := range sensitiveParams {
+			if _, apiOk := params[sp]; apiOk {
+				if _, exists := d.GetOkExists("sensitive_params.0." + sp); exists {
+					delete(params, sp)
+				} else {
+					params[sp] = d.Get("params." + sp)
+				}
+			}
+		}
+	}
+
+	return res, nil
+}

website/docs/r/bigquery_data_transfer_config.html.markdown

@@ -32,6 +32,9 @@ To get more information about Config, see:
 * How-to Guides
     * [Official Documentation](https://cloud.google.com/bigquery/docs/reference/datatransfer/rest/)
 
+~> **Warning:** All arguments including `sensitive_params.secret_access_key` will be stored in the raw
+state as plain-text. [Read more about sensitive data in state](/docs/state/sensitive-data.html).
+
 ## Example Usage - Bigquerydatatransfer Config Scheduled Query
 
 
@@ -122,6 +125,16 @@ The following arguments are supported:
   (Optional)
   When set to true, no runs are scheduled for a given transfer.
 
+* `sensitive_params` -
+  (Optional)
+  Different parameters are configured primarily using the the `params` field on this
+  resource. This block contains the parameters which contain secrets or passwords so that they can be marked
+  sensitive and hidden from plan output. The name of the field, eg: secret_access_key, will be the key
+  in the `params` map in the api request.
+  Credentials may not be specified in both locations and will cause an error. Changing from one location
+  to a different credential configuration in the config will require an apply to update state.
+  Structure is documented below.
+
 * `location` -
   (Optional)
   The geographic location where the transfer config should reside.
@@ -137,6 +150,13 @@ The following arguments are supported:
     If it is not provided, the provider project is used.
 
 
+The `sensitive_params` block supports:
+
+* `secret_access_key` -
+  (Required)
+  The Secret Access Key of the AWS account transferring data from.
+  **Note**: This property is sensitive and will not be displayed in the plan.
+
 ## Attributes Reference
 
 In addition to the arguments listed above, the following computed attributes are exported: