introduce DecodeMapstructure to allow type to define custom decode logic

Signed-off-by: Nicolas De Loof <[email protected]>

Author: ndeloof

loader/interpolate.go

@@ -46,10 +46,6 @@ var interpolateTypeCastMapping = map[tree.Path]interp.Cast{
 	servicePath("deploy", "placement", "max_replicas_per_node"):    toInt,
 	servicePath("healthcheck", "retries"):                          toInt,
 	servicePath("healthcheck", "disable"):                          toBoolean,
-	servicePath("mem_limit"):                                       toUnitBytes,
-	servicePath("mem_reservation"):                                 toUnitBytes,
-	servicePath("memswap_limit"):                                   toUnitBytes,
-	servicePath("mem_swappiness"):                                  toUnitBytes,
 	servicePath("oom_kill_disable"):                                toBoolean,
 	servicePath("oom_score_adj"):                                   toInt64,
 	servicePath("pids_limit"):                                      toInt64,
@@ -58,16 +54,13 @@ var interpolateTypeCastMapping = map[tree.Path]interp.Cast{
 	servicePath("read_only"):                                       toBoolean,
 	servicePath("scale"):                                           toInt,
 	servicePath("secrets", tree.PathMatchList, "mode"):             toInt,
-	servicePath("shm_size"):                                        toUnitBytes,
 	servicePath("stdin_open"):                                      toBoolean,
-	servicePath("stop_grace_period"):                               toDuration,
 	servicePath("tty"):                                             toBoolean,
 	servicePath("ulimits", tree.PathMatchAll):                      toInt,
 	servicePath("ulimits", tree.PathMatchAll, "hard"):              toInt,
 	servicePath("ulimits", tree.PathMatchAll, "soft"):              toInt,
 	servicePath("volumes", tree.PathMatchList, "read_only"):        toBoolean,
 	servicePath("volumes", tree.PathMatchList, "volume", "nocopy"): toBoolean,
-	servicePath("volumes", tree.PathMatchList, "tmpfs", "size"):    toUnitBytes,
 	iPath("networks", tree.PathMatchAll, "external"):               toBoolean,
 	iPath("networks", tree.PathMatchAll, "internal"):               toBoolean,
 	iPath("networks", tree.PathMatchAll, "attachable"):             toBoolean,
@@ -93,14 +86,6 @@ func toInt64(value string) (interface{}, error) {
 	return strconv.ParseInt(value, 10, 64)
 }
 
-func toUnitBytes(value string) (interface{}, error) {
-	return transformSize(value)
-}
-
-func toDuration(value string) (interface{}, error) {
-	return transformStringToDuration(value)
-}
-
 func toFloat(value string) (interface{}, error) {
 	return strconv.ParseFloat(value, 64)
 }

loader/loader.go

@@ -28,15 +28,12 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/compose-spec/compose-go/consts"
 	interp "github.com/compose-spec/compose-go/interpolation"
 	"github.com/compose-spec/compose-go/schema"
 	"github.com/compose-spec/compose-go/template"
 	"github.com/compose-spec/compose-go/types"
-	"github.com/docker/go-units"
-	"github.com/mattn/go-shellwords"
 	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -586,7 +583,7 @@ func Transform(source interface{}, target interface{}, additionalTransformers ..
 	config := &mapstructure.DecoderConfig{
 		DecodeHook: mapstructure.ComposeDecodeHookFunc(
 			createTransformHook(additionalTransformers...),
-			mapstructure.StringToTimeDurationHookFunc()),
+			decoderHook),
 		Result:   target,
 		TagName:  "yaml",
 		Metadata: &data,
@@ -611,11 +608,9 @@ func createTransformHook(additionalTransformers ...Transformer) mapstructure.Dec
 	transforms := map[reflect.Type]func(interface{}) (interface{}, error){
 		reflect.TypeOf(types.External{}):                         transformExternal,
 		reflect.TypeOf(types.HealthCheckTest{}):                  transformHealthCheckTest,
-		reflect.TypeOf(types.ShellCommand{}):                     transformShellCommand,
 		reflect.TypeOf(types.StringList{}):                       transformStringList,
 		reflect.TypeOf(types.Options{}):                          transformMapStringString,
 		reflect.TypeOf(types.UlimitsConfig{}):                    transformUlimits,
-		reflect.TypeOf(types.UnitBytes(0)):                       transformSize,
 		reflect.TypeOf([]types.ServicePortConfig{}):              transformServicePort,
 		reflect.TypeOf(types.ServiceSecretConfig{}):              transformFileReferenceConfig,
 		reflect.TypeOf(types.ServiceConfigObjConfig{}):           transformFileReferenceConfig,
@@ -628,7 +623,6 @@ func createTransformHook(additionalTransformers ...Transformer) mapstructure.Dec
 		reflect.TypeOf(types.HostsList{}):                        transformMappingOrListFunc(":", false),
 		reflect.TypeOf(types.ServiceVolumeConfig{}):              transformServiceVolumeConfig,
 		reflect.TypeOf(types.BuildConfig{}):                      transformBuildConfig,
-		reflect.TypeOf(types.Duration(0)):                        transformStringToDuration,
 		reflect.TypeOf(types.DependsOnConfig{}):                  transformDependsOnConfig,
 		reflect.TypeOf(types.ExtendsConfig{}):                    transformExtendsConfig,
 		reflect.TypeOf(types.DeviceRequest{}):                    transformServiceDeviceRequest,
@@ -1312,13 +1306,6 @@ func transformValueToMapEntry(value string, separator string, allowNil bool) (st
 	}
 }
 
-var transformShellCommand TransformerFunc = func(value interface{}) (interface{}, error) {
-	if str, ok := value.(string); ok {
-		return shellwords.Parse(str)
-	}
-	return value, nil
-}
-
 var transformHealthCheckTest TransformerFunc = func(data interface{}) (interface{}, error) {
 	switch value := data.(type) {
 	case string:
@@ -1330,34 +1317,6 @@ var transformHealthCheckTest TransformerFunc = func(data interface{}) (interface
 	}
 }
 
-var transformSize TransformerFunc = func(value interface{}) (interface{}, error) {
-	switch value := value.(type) {
-	case int:
-		return int64(value), nil
-	case int64, types.UnitBytes:
-		return value, nil
-	case string:
-		return units.RAMInBytes(value)
-	default:
-		return value, errors.Errorf("invalid type for size %T", value)
-	}
-}
-
-var transformStringToDuration TransformerFunc = func(value interface{}) (interface{}, error) {
-	switch value := value.(type) {
-	case string:
-		d, err := time.ParseDuration(value)
-		if err != nil {
-			return value, err
-		}
-		return types.Duration(d), nil
-	case types.Duration:
-		return value, nil
-	default:
-		return value, errors.Errorf("invalid type %T for duration", value)
-	}
-}
-
 func toMapStringString(value map[string]interface{}, allowNil bool) map[string]interface{} {
 	output := make(map[string]interface{})
 	for key, value := range value {

loader/mapstructure.go

@@ -0,0 +1,53 @@
+/*
+   Copyright 2020 The Compose Specification Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package loader
+
+import "reflect"
+
+// comparable to yaml.Unmarshaler, decoder allow a type to define it's own custom logic to convert value
+// see https://github.com/mitchellh/mapstructure/pull/294
+type decoder interface {
+	DecodeMapstructure(interface{}) error
+}
+
+// see https://github.com/mitchellh/mapstructure/issues/115#issuecomment-735287466
+// adapted to support types derived from built-in types, as DecodeMapstructure would not be able to mutate internal
+// value, so need to invoke DecodeMapstructure defined by pointer to type
+func decoderHook(from reflect.Value, to reflect.Value) (interface{}, error) {
+	// If the destination implements the decoder interface
+	u, ok := to.Interface().(decoder)
+	if !ok {
+		// for non-struct types we need to invoke func (*type) DecodeMapstructure()
+		if to.CanAddr() {
+			pto := to.Addr()
+			u, ok = pto.Interface().(decoder)
+		}
+		if !ok {
+			return from.Interface(), nil
+		}
+	}
+	// If it is nil and a pointer, create and assign the target value first
+	if to.Type().Kind() == reflect.Ptr && to.IsNil() {
+		to.Set(reflect.New(to.Type().Elem()))
+		u = to.Interface().(decoder)
+	}
+	// Call the custom DecodeMapstructure method
+	if err := u.DecodeMapstructure(from.Interface()); err != nil {
+		return to.Interface(), err
+	}
+	return to.Interface(), nil
+}

loader/mapstructure_test.go

@@ -0,0 +1,65 @@
+/*
+   Copyright 2020 The Compose Specification Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package loader
+
+import (
+	"testing"
+
+	"github.com/compose-spec/compose-go/types"
+	"github.com/mitchellh/mapstructure"
+	"gotest.tools/v3/assert"
+)
+
+func TestDecodeMapStructure(t *testing.T) {
+	var target types.ServiceConfig
+	data := mapstructure.Metadata{}
+	config := &mapstructure.DecoderConfig{
+		Result:     &target,
+		TagName:    "yaml",
+		Metadata:   &data,
+		DecodeHook: mapstructure.ComposeDecodeHookFunc(decoderHook),
+	}
+	decoder, err := mapstructure.NewDecoder(config)
+	assert.NilError(t, err)
+	err = decoder.Decode(map[string]interface{}{
+		"mem_limit":         "640k",
+		"command":           "echo hello",
+		"stop_grace_period": "60s",
+		"labels": []interface{}{
+			"FOO=BAR",
+		},
+		"deploy": map[string]interface{}{
+			"labels": map[string]interface{}{
+				"FOO": "BAR",
+				"BAZ": nil,
+				"QIX": 2,
+				"ZOT": true,
+			},
+		},
+	})
+	assert.NilError(t, err)
+	assert.Equal(t, target.MemLimit, types.UnitBytes(640*1024))
+	assert.DeepEqual(t, target.Command, types.ShellCommand{"echo", "hello"})
+	assert.Equal(t, *target.StopGracePeriod, types.Duration(60_000_000_000))
+	assert.DeepEqual(t, target.Labels, types.Labels{"FOO": "BAR"})
+	assert.DeepEqual(t, target.Deploy.Labels, types.Labels{
+		"FOO": "BAR",
+		"BAZ": "",
+		"QIX": "2",
+		"ZOT": "true",
+	})
+}

types/bytes.go

@@ -0,0 +1,42 @@
+/*
+   Copyright 2020 The Compose Specification Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package types
+
+import (
+	"fmt"
+
+	"github.com/docker/go-units"
+)
+
+// UnitBytes is the bytes type
+type UnitBytes int64
+
+// MarshalYAML makes UnitBytes implement yaml.Marshaller
+func (u UnitBytes) MarshalYAML() (interface{}, error) {
+	return fmt.Sprintf("%d", u), nil
+}
+
+// MarshalJSON makes UnitBytes implement json.Marshaler
+func (u UnitBytes) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`"%d"`, u)), nil
+}
+
+func (u *UnitBytes) DecodeMapstructure(value interface{}) error {
+	v, err := units.RAMInBytes(fmt.Sprint(value))
+	*u = UnitBytes(v)
+	return err
+}

types/command.go

@@ -0,0 +1,86 @@
+/*
+   Copyright 2020 The Compose Specification Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package types
+
+import "github.com/mattn/go-shellwords"
+
+// ShellCommand is a string or list of string args.
+//
+// When marshaled to YAML, nil command fields will be omitted if `omitempty`
+// is specified as a struct tag. Explicitly empty commands (i.e. `[]` or
+// empty string will serialize to an empty array (`[]`).
+//
+// When marshaled to JSON, the `omitempty` struct must NOT be specified.
+// If the command field is nil, it will be serialized as `null`.
+// Explicitly empty commands (i.e. `[]` or empty string) will serialize to
+// an empty array (`[]`).
+//
+// The distinction between nil and explicitly empty is important to distinguish
+// between an unset value and a provided, but empty, value, which should be
+// preserved so that it can override any base value (e.g. container entrypoint).
+//
+// The different semantics between YAML and JSON are due to limitations with
+// JSON marshaling + `omitempty` in the Go stdlib, while gopkg.in/yaml.v3 gives
+// us more flexibility via the yaml.IsZeroer interface.
+//
+// In the future, it might make sense to make fields of this type be
+// `*ShellCommand` to avoid this situation, but that would constitute a
+// breaking change.
+type ShellCommand []string
+
+// IsZero returns true if the slice is nil.
+//
+// Empty (but non-nil) slices are NOT considered zero values.
+func (s ShellCommand) IsZero() bool {
+	// we do NOT want len(s) == 0, ONLY explicitly nil
+	return s == nil
+}
+
+// MarshalYAML returns nil (which will be serialized as `null`) for nil slices
+// and delegates to the standard marshaller behavior otherwise.
+//
+// NOTE: Typically the nil case here is not hit because IsZero has already
+// short-circuited marshalling, but this ensures that the type serializes
+// accurately if the `omitempty` struct tag is omitted/forgotten.
+//
+// A similar MarshalJSON() implementation is not needed because the Go stdlib
+// already serializes nil slices to `null`, whereas gopkg.in/yaml.v3 by default
+// serializes nil slices to `[]`.
+func (s ShellCommand) MarshalYAML() (interface{}, error) {
+	if s == nil {
+		return nil, nil
+	}
+	return []string(s), nil
+}
+
+func (s *ShellCommand) DecodeMapstructure(value interface{}) error {
+	switch v := value.(type) {
+	case string:
+		cmd, err := shellwords.Parse(v)
+		if err != nil {
+			return err
+		}
+		*s = cmd
+	case []interface{}:
+		cmd := make([]string, len(v))
+		for i, s := range v {
+			cmd[i] = s.(string)
+		}
+		*s = cmd
+	}
+	return nil
+}