Forward to 1.7.0

CHANGELOG.md

@@ -1,6 +1,12 @@
+# 1.7.0 (Unreleased)
+
+* `cty`: `Value.UnmarkDeepWithPaths` and `Value.MarkWithPaths` are like `Value.UnmarkDeep` and `Value.Mark` but they retain path information for each marked value, so that marks can be re-applied later without all the loss of detail that results from `Value.UnmarkDeep` aggregating together all of the nested marks.
+* `function`: Unless a parameter has `AllowMarks: true` explicitly set, the functions infrastructure will now guarantee that it never sees a marked value even if the mark is deep inside a data structure. Previously that guarantee was only shallow for the top-level value, similar to `AllowUnknown`, but because marks are a relatively new addition to `cty` and numerous existing functions are not written to deal with them this is the more conservative and robust default. ([#72](https://github.com/zclconf/go-cty/pull/72))
+* `function/stdlib`: The `formatdate` function was not correctly handling literal sequences at the end of the format string. It will now handle those as intended. ([#69](https://github.com/zclconf/go-cty/pull/69))
+
 # 1.6.1 (Unreleased)
 
-* Fix a regression from 1.6.0 where `Value.RawEqual` no longer returned the correct result given a pair of sets containing partially-unknown values. ([#64](https://github.com/zclconf/go-cty/pull/64))
+* `cty`:: Fix a regression from 1.6.0 where `Value.RawEqual` no longer returned the correct result given a pair of sets containing partially-unknown values. ([#64](https://github.com/zclconf/go-cty/pull/64))
 
 # 1.6.0 (Unreleased)
 

cty/function/function.go

@@ -244,19 +244,21 @@ func (f Function) Call(args []cty.Value) (val cty.Value, err error) {
 			return cty.UnknownVal(expectedType), nil
 		}
 
-		if val.IsMarked() && !spec.AllowMarked {
-			unwrappedVal, marks := val.Unmark()
-			// In order to avoid additional overhead on applications that
-			// are not using marked values, we copy the given args only
-			// if we encounter a marked value we need to unmark. However,
-			// as a consequence we end up doing redundant copying if multiple
-			// marked values need to be unwrapped. That seems okay because
-			// argument lists are generally small.
-			newArgs := make([]cty.Value, len(args))
-			copy(newArgs, args)
-			newArgs[i] = unwrappedVal
-			resultMarks = append(resultMarks, marks)
-			args = newArgs
+		if !spec.AllowMarked {
+			unwrappedVal, marks := val.UnmarkDeep()
+			if len(marks) > 0 {
+				// In order to avoid additional overhead on applications that
+				// are not using marked values, we copy the given args only
+				// if we encounter a marked value we need to unmark. However,
+				// as a consequence we end up doing redundant copying if multiple
+				// marked values need to be unwrapped. That seems okay because
+				// argument lists are generally small.
+				newArgs := make([]cty.Value, len(args))
+				copy(newArgs, args)
+				newArgs[i] = unwrappedVal
+				resultMarks = append(resultMarks, marks)
+				args = newArgs
+			}
 		}
 	}
 
@@ -266,13 +268,15 @@ func (f Function) Call(args []cty.Value) (val cty.Value, err error) {
 			if !val.IsKnown() && !spec.AllowUnknown {
 				return cty.UnknownVal(expectedType), nil
 			}
-			if val.IsMarked() && !spec.AllowMarked {
-				unwrappedVal, marks := val.Unmark()
-				newArgs := make([]cty.Value, len(args))
-				copy(newArgs, args)
-				newArgs[len(posArgs)+i] = unwrappedVal
-				resultMarks = append(resultMarks, marks)
-				args = newArgs
+			if !spec.AllowMarked {
+				unwrappedVal, marks := val.UnmarkDeep()
+				if len(marks) > 0 {
+					newArgs := make([]cty.Value, len(args))
+					copy(newArgs, args)
+					newArgs[len(posArgs)+i] = unwrappedVal
+					resultMarks = append(resultMarks, marks)
+					args = newArgs
+				}
 			}
 		}
 	}

cty/function/stdlib/datetime.go

@@ -362,9 +362,14 @@ func splitDateFormat(data []byte, atEOF bool) (advance int, token []byte, err er
 		for i := 1; i < len(data); i++ {
 			if data[i] == esc {
 				if (i + 1) == len(data) {
-					// We need at least one more byte to decide if this is an
-					// escape or a terminator.
-					return 0, nil, nil
+					if atEOF {
+						// We have a closing quote and are at the end of our input
+						return len(data), data, nil
+					} else {
+						// We need at least one more byte to decide if this is an
+						// escape or a terminator.
+						return 0, nil, nil
+					}
 				}
 				if data[i+1] == esc {
 					i++ // doubled-up quotes are an escape sequence

cty/function/stdlib/datetime_test.go

@@ -39,6 +39,11 @@ func TestFormatDate(t *testing.T) {
 			cty.StringVal("3 o'clock PM"),
 			``,
 		},
+		{
+			cty.StringVal("H 'o''clock'"),
+			cty.StringVal("3 o'clock"),
+			``,
+		},
 		{
 			cty.StringVal("hh:mm:ssZZZZ"),
 			cty.StringVal("15:04:05+0000"),

cty/function/stdlib/string_test.go

@@ -393,3 +393,83 @@ func TestSubstr(t *testing.T) {
 		})
 	}
 }
+
+func TestJoin(t *testing.T) {
+	tests := map[string]struct {
+		Separator cty.Value
+		Lists     []cty.Value
+		Want      cty.Value
+	}{
+		"single two-element list": {
+			cty.StringVal("-"),
+			[]cty.Value{
+				cty.ListVal([]cty.Value{cty.StringVal("hello"), cty.StringVal("world")}),
+			},
+			cty.StringVal("hello-world"),
+		},
+		"multiple single-element lists": {
+			cty.StringVal("-"),
+			[]cty.Value{
+				cty.ListVal([]cty.Value{cty.StringVal("chicken")}),
+				cty.ListVal([]cty.Value{cty.StringVal("egg")}),
+			},
+			cty.StringVal("chicken-egg"),
+		},
+		"single single-element list": {
+			cty.StringVal("-"),
+			[]cty.Value{
+				cty.ListVal([]cty.Value{cty.StringVal("chicken")}),
+			},
+			cty.StringVal("chicken"),
+		},
+		"blank separator": {
+			cty.StringVal(""),
+			[]cty.Value{
+				cty.ListVal([]cty.Value{cty.StringVal("horse"), cty.StringVal("face")}),
+			},
+			cty.StringVal("horseface"),
+		},
+		"marked list": {
+			cty.StringVal("-"),
+			[]cty.Value{
+				cty.ListVal([]cty.Value{cty.StringVal("hello"), cty.StringVal("world")}).Mark("sensitive"),
+			},
+			cty.StringVal("hello-world").Mark("sensitive"),
+		},
+		"marked separator": {
+			cty.StringVal("-").Mark("sensitive"),
+			[]cty.Value{
+				cty.ListVal([]cty.Value{cty.StringVal("hello"), cty.StringVal("world")}),
+			},
+			cty.StringVal("hello-world").Mark("sensitive"),
+		},
+		"list with some marked elements": {
+			cty.StringVal("-"),
+			[]cty.Value{
+				cty.ListVal([]cty.Value{cty.StringVal("hello").Mark("sensitive"), cty.StringVal("world")}),
+			},
+			cty.StringVal("hello-world").Mark("sensitive"),
+		},
+		"multiple marks": {
+			cty.StringVal("-").Mark("a"),
+			[]cty.Value{
+				cty.ListVal([]cty.Value{cty.StringVal("hello").Mark("b"), cty.StringVal("world").Mark("c")}),
+			},
+			cty.StringVal("hello-world").WithMarks(cty.NewValueMarks("a", "b", "c")),
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			got, err := Join(test.Separator, test.Lists...)
+
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+
+			if !got.RawEquals(test.Want) {
+				t.Errorf("wrong result\ngot:  %#v\nwant: %#v", got, test.Want)
+			}
+		})
+	}
+}

cty/json/marshal.go

@@ -10,7 +10,7 @@ import (
 
 func marshal(val cty.Value, t cty.Type, path cty.Path, b *bytes.Buffer) error {
 	if val.IsMarked() {
-		return path.NewErrorf("value has marks, so it cannot be seralized")
+		return path.NewErrorf("value has marks, so it cannot be serialized as JSON")
 	}
 
 	// If we're going to decode as DynamicPseudoType then we need to save

cty/marks.go

@@ -67,6 +67,23 @@ func (m ValueMarks) GoString() string {
 	return s.String()
 }
 
+// PathValueMarks is a structure that enables tracking marks
+// and the paths where they are located in one type
+type PathValueMarks struct {
+	Path  Path
+	Marks ValueMarks
+}
+
+func (p PathValueMarks) Equal(o PathValueMarks) bool {
+	if !p.Path.Equals(o.Path) {
+		return false
+	}
+	if !p.Marks.Equal(o.Marks) {
+		return false
+	}
+	return true
+}
+
 // IsMarked returns true if and only if the receiving value carries at least
 // one mark. A marked value cannot be used directly with integration methods
 // without explicitly unmarking it (and retrieving the markings) first.
@@ -174,6 +191,31 @@ func (val Value) Mark(mark interface{}) Value {
 	}
 }
 
+type applyPathValueMarksTransformer struct {
+	pvm []PathValueMarks
+}
+
+func (t *applyPathValueMarksTransformer) Enter(p Path, v Value) (Value, error) {
+	return v, nil
+}
+
+func (t *applyPathValueMarksTransformer) Exit(p Path, v Value) (Value, error) {
+	for _, path := range t.pvm {
+		if p.Equals(path.Path) {
+			return v.WithMarks(path.Marks), nil
+		}
+	}
+	return v, nil
+}
+
+// MarkWithPaths accepts a slice of PathValueMarks to apply
+// markers to particular paths and returns the marked
+// Value.
+func (val Value) MarkWithPaths(pvm []PathValueMarks) Value {
+	ret, _ := TransformWithTransformer(val, &applyPathValueMarksTransformer{pvm})
+	return ret
+}
+
 // Unmark separates the marks of the receiving value from the value itself,
 // removing a new unmarked value and a map (representing a set) of the marks.
 //
@@ -191,24 +233,54 @@ func (val Value) Unmark() (Value, ValueMarks) {
 	}, marks
 }
 
+type unmarkTransformer struct {
+	pvm []PathValueMarks
+}
+
+func (t *unmarkTransformer) Enter(p Path, v Value) (Value, error) {
+	unmarkedVal, marks := v.Unmark()
+	if len(marks) > 0 {
+		path := make(Path, len(p), len(p)+1)
+		copy(path, p)
+		t.pvm = append(t.pvm, PathValueMarks{path, marks})
+	}
+	return unmarkedVal, nil
+}
+
+func (t *unmarkTransformer) Exit(p Path, v Value) (Value, error) {
+	return v, nil
+}
+
 // UnmarkDeep is similar to Unmark, but it works with an entire nested structure
 // rather than just the given value directly.
 //
 // The result is guaranteed to contain no nested values that are marked, and
 // the returned marks set includes the superset of all of the marks encountered
 // during the operation.
 func (val Value) UnmarkDeep() (Value, ValueMarks) {
+	t := unmarkTransformer{}
+	ret, _ := TransformWithTransformer(val, &t)
+
 	marks := make(ValueMarks)
-	ret, _ := Transform(val, func(_ Path, v Value) (Value, error) {
-		unmarkedV, valueMarks := v.Unmark()
-		for m, s := range valueMarks {
+	for _, pvm := range t.pvm {
+		for m, s := range pvm.Marks {
 			marks[m] = s
 		}
-		return unmarkedV, nil
-	})
+	}
+
 	return ret, marks
 }
 
+// UnmarkDeepWithPaths is like UnmarkDeep, except it returns a slice
+// of PathValueMarks rather than a superset of all marks. This allows
+// a caller to know which marks are associated with which paths
+// in the Value.
+func (val Value) UnmarkDeepWithPaths() (Value, []PathValueMarks) {
+	t := unmarkTransformer{}
+	ret, _ := TransformWithTransformer(val, &t)
+	return ret, t.pvm
+}
+
 func (val Value) unmarkForce() Value {
 	unw, _ := val.Unmark()
 	return unw