sanitize: allow for the sanitization of sensitive values

This adds a new package and functions for sanitization of values marked
as sensitive in the plan, where we can get particular data to do it.

This data is derived in a number of ways, also documented in the
top-level SanitizePlan function:

* ResourceChanges are sanitized based on BeforeSensitive and
AfterSensitive fields.

* Variables are sanitized based on variable config data found in the
root module of the Config.

* PlannedValues are sanitized based on the values found in
AfterSensitive in ResourceChanges. Outputs are sanitized according to
the appropriate sensitivity flags provided for the output.

* PriorState is sanitized based on the values found in BeforeSensitive
in ResourceChanges. Outputs are sanitized according to the appropriate
sensitivity flags provided for the output.

* OutputChanges are sanitized based on the values found in
BeforeSensitive and AfterSensitive. This generally means that any
sensitive output will have OutputChange fully obfuscated as the
BeforeSensitive and AfterSensitive in outputs are opaquely the same.

Author: vancluever

go.mod

@@ -5,5 +5,7 @@ go 1.13
 require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/google/go-cmp v0.3.1
+	github.com/mitchellh/copystructure v1.2.0
+	github.com/sebdah/goldie v1.0.0
 	github.com/zclconf/go-cty v1.2.1
 )

go.sum

@@ -1,4 +1,5 @@
 github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -8,6 +9,17 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
+github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
+github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
+github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
+github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sebdah/goldie v1.0.0 h1:9GNhIat69MSlz/ndaBg48vl9dF5fI+NBB6kfOxgfkMc=
+github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
 github.com/zclconf/go-cty v1.2.1 h1:vGMsygfmeCl4Xb6OA5U5XVAaQZ69FvoG7X2jUtQujb8=
 github.com/zclconf/go-cty v1.2.1/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8=

sanitize/copy.go

@@ -0,0 +1,74 @@
+package sanitize
+
+import (
+	"reflect"
+
+	tfjson "github.com/hashicorp/terraform-json"
+	"github.com/mitchellh/copystructure"
+)
+
+// copyStructureCopy is an internal function that wraps copystructure.Copy with
+// a shallow copier for unknown values.
+//
+// Performing the shallow copy of the unknown values is important
+// here, as unknown values are parsed in with the main terraform-json
+// package as singletons, and must continue to be comparable.
+func copyStructureCopy(v interface{}) (interface{}, error) {
+	c := &copystructure.Config{
+		ShallowCopiers: map[reflect.Type]struct{}{
+			reflect.TypeOf(tfjson.UnknownConstantValue): struct{}{},
+		},
+	}
+
+	return c.Copy(v)
+}
+
+// copyChange copies a Change value and returns the copy.
+func copyChange(old *tfjson.Change) (*tfjson.Change, error) {
+	c, err := copyStructureCopy(old)
+	if err != nil {
+		return nil, err
+	}
+
+	return c.(*tfjson.Change), nil
+}
+
+// copyPlan copies a Plan value and returns the copy.
+func copyPlan(old *tfjson.Plan) (*tfjson.Plan, error) {
+	c, err := copyStructureCopy(old)
+	if err != nil {
+		return nil, err
+	}
+
+	return c.(*tfjson.Plan), nil
+}
+
+// copyPlanVariable copies a PlanVariable value and returns the copy.
+func copyPlanVariable(old *tfjson.PlanVariable) (*tfjson.PlanVariable, error) {
+	c, err := copyStructureCopy(old)
+	if err != nil {
+		return nil, err
+	}
+
+	return c.(*tfjson.PlanVariable), nil
+}
+
+// copyStateResource copies a StateResource value and returns the copy.
+func copyStateResource(old *tfjson.StateResource) (*tfjson.StateResource, error) {
+	c, err := copyStructureCopy(old)
+	if err != nil {
+		return nil, err
+	}
+
+	return c.(*tfjson.StateResource), nil
+}
+
+// copyStateOutput copies a StateOutput value and returns the copy.
+func copyStateOutputs(old map[string]*tfjson.StateOutput) (map[string]*tfjson.StateOutput, error) {
+	c, err := copystructure.Copy(old)
+	if err != nil {
+		return nil, err
+	}
+
+	return c.(map[string]*tfjson.StateOutput), nil
+}

sanitize/copy_test.go

@@ -0,0 +1,19 @@
+package sanitize
+
+import (
+	"testing"
+
+	tfjson "github.com/hashicorp/terraform-json"
+)
+
+func TestCopyStructureCopy(t *testing.T) {
+	in := tfjson.UnknownConstantValue
+	out, err := copyStructureCopy(in)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if in != out {
+		t.Fatal("did not shallow copy")
+	}
+}

sanitize/sanitize_change.go

@@ -0,0 +1,53 @@
+package sanitize
+
+import (
+	tfjson "github.com/hashicorp/terraform-json"
+)
+
+// SanitizeChange traverses a Change and replaces all values at
+// the particular locations marked by BeforeSensitive AfterSensitive
+// with the value supplied as replaceWith.
+//
+// A new change is issued.
+func SanitizeChange(old *tfjson.Change, replaceWith interface{}) (*tfjson.Change, error) {
+	result, err := copyChange(old)
+	if err != nil {
+		return nil, err
+	}
+
+	result.Before = sanitizeChangeValue(result.Before, result.BeforeSensitive, replaceWith)
+	result.After = sanitizeChangeValue(result.After, result.AfterSensitive, replaceWith)
+
+	return result, nil
+}
+
+func sanitizeChangeValue(old, sensitive, replaceWith interface{}) interface{} {
+	// Only expect deep types that we would normally see in JSON, so
+	// arrays and objects.
+	switch x := old.(type) {
+	case []interface{}:
+		if filterSlice, ok := sensitive.([]interface{}); ok {
+			for i := range filterSlice {
+				if i >= len(x) {
+					break
+				}
+
+				x[i] = sanitizeChangeValue(x[i], filterSlice[i], replaceWith)
+			}
+		}
+	case map[string]interface{}:
+		if filterMap, ok := sensitive.(map[string]interface{}); ok {
+			for filterKey := range filterMap {
+				if value, ok := x[filterKey]; ok {
+					x[filterKey] = sanitizeChangeValue(value, filterMap[filterKey], replaceWith)
+				}
+			}
+		}
+	}
+
+	if shouldFilter, ok := sensitive.(bool); ok && shouldFilter {
+		return replaceWith
+	}
+
+	return old
+}

sanitize/sanitize_change_test.go

@@ -0,0 +1,142 @@
+package sanitize
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	tfjson "github.com/hashicorp/terraform-json"
+)
+
+type testChangeCase struct {
+	name     string
+	old      *tfjson.Change
+	expected *tfjson.Change
+}
+
+func changeCases() []testChangeCase {
+	return []testChangeCase{
+		{
+			name: "basic",
+			old: &tfjson.Change{
+				Before: map[string]interface{}{
+					"foo": map[string]interface{}{"a": "foo"},
+					"bar": map[string]interface{}{"a": "foo"},
+					"baz": map[string]interface{}{"a": "foo"},
+					"qux": map[string]interface{}{
+						"a": map[string]interface{}{
+							"b": "foo",
+						},
+						"c": "bar",
+					},
+					"quxx": map[string]interface{}{
+						"a": map[string]interface{}{
+							"b": "foo",
+						},
+						"c": "bar",
+					},
+				},
+				After: map[string]interface{}{
+					"one":   map[string]interface{}{"x": "one"},
+					"two":   map[string]interface{}{"x": "one"},
+					"three": map[string]interface{}{"x": "one"},
+					"four": map[string]interface{}{
+						"x": map[string]interface{}{
+							"y": "one",
+						},
+						"z": "two",
+					},
+					"five": map[string]interface{}{
+						"x": map[string]interface{}{
+							"y": "one",
+						},
+						"z": "two",
+					},
+				},
+				BeforeSensitive: map[string]interface{}{
+					"foo":  map[string]interface{}{},
+					"bar":  true,
+					"baz":  map[string]interface{}{"a": true},
+					"qux":  map[string]interface{}{},
+					"quxx": map[string]interface{}{"c": true},
+				},
+				AfterSensitive: map[string]interface{}{
+					"one":   map[string]interface{}{},
+					"two":   true,
+					"three": map[string]interface{}{"x": true},
+					"four":  map[string]interface{}{},
+					"five":  map[string]interface{}{"z": true},
+				},
+			},
+			expected: &tfjson.Change{
+				Before: map[string]interface{}{
+					"foo": map[string]interface{}{"a": "foo"},
+					"bar": DefaultSensitiveValue,
+					"baz": map[string]interface{}{"a": DefaultSensitiveValue},
+					"qux": map[string]interface{}{
+						"a": map[string]interface{}{
+							"b": "foo",
+						},
+						"c": "bar",
+					},
+					"quxx": map[string]interface{}{
+						"a": map[string]interface{}{
+							"b": "foo",
+						},
+						"c": DefaultSensitiveValue,
+					},
+				},
+				After: map[string]interface{}{
+					"one":   map[string]interface{}{"x": "one"},
+					"two":   DefaultSensitiveValue,
+					"three": map[string]interface{}{"x": DefaultSensitiveValue},
+					"four": map[string]interface{}{
+						"x": map[string]interface{}{
+							"y": "one",
+						},
+						"z": "two",
+					},
+					"five": map[string]interface{}{
+						"x": map[string]interface{}{
+							"y": "one",
+						},
+						"z": DefaultSensitiveValue,
+					},
+				},
+				BeforeSensitive: map[string]interface{}{
+					"foo":  map[string]interface{}{},
+					"bar":  true,
+					"baz":  map[string]interface{}{"a": true},
+					"qux":  map[string]interface{}{},
+					"quxx": map[string]interface{}{"c": true},
+				},
+				AfterSensitive: map[string]interface{}{
+					"one":   map[string]interface{}{},
+					"two":   true,
+					"three": map[string]interface{}{"x": true},
+					"four":  map[string]interface{}{},
+					"five":  map[string]interface{}{"z": true},
+				},
+			},
+		},
+	}
+}
+
+func TestSanitizeChange(t *testing.T) {
+	for i, tc := range changeCases() {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			actual, err := SanitizeChange(tc.old, DefaultSensitiveValue)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if diff := cmp.Diff(tc.expected, actual); diff != "" {
+				t.Errorf("SanitizeChange() mismatch (-expected +actual):\n%s", diff)
+			}
+
+			if diff := cmp.Diff(changeCases()[i].old, tc.old); diff != "" {
+				t.Errorf("SanitizeChange() altered original (-expected +actual):\n%s", diff)
+			}
+		})
+	}
+}