Auto parameters

cmd/cnab-run/inspect.go

@@ -1,14 +1,21 @@
 package main
 
 import (
+	"bytes"
 	"os"
 
 	appinspect "github.com/docker/app/internal/inspect"
 	"github.com/docker/app/internal/packager"
+	"github.com/docker/app/types"
+	"github.com/pkg/errors"
 )
 
 func inspectAction(instanceName string) error {
-	app, err := packager.Extract("")
+	overrides, err := parseOverrides()
+	if err != nil {
+		return errors.Wrap(err, "unable to parse auto-parameter values")
+	}
+	app, err := packager.Extract("", types.WithComposes(bytes.NewReader(overrides)))
 	// todo: merge additional compose file
 	if err != nil {
 		return err

cmd/cnab-run/install.go

@@ -1,21 +1,26 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strconv"
+	"strings"
 
 	"github.com/deislabs/duffle/pkg/bundle"
 	"github.com/docker/app/internal"
 	"github.com/docker/app/internal/packager"
 	"github.com/docker/app/render"
+	"github.com/docker/app/types"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/cli/cli/command/stack"
 	"github.com/docker/cli/cli/command/stack/options"
 	"github.com/docker/cli/cli/command/stack/swarm"
 	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
+	yaml "gopkg.in/yaml.v2"
 )
 
 const (
@@ -29,7 +34,11 @@ func installAction(instanceName string) error {
 	if err != nil {
 		return errors.Wrap(err, "unable to restore docker context")
 	}
-	app, err := packager.Extract("")
+	overrides, err := parseOverrides()
+	if err != nil {
+		return errors.Wrap(err, "unable to parse auto-parameter values")
+	}
+	app, err := packager.Extract("", types.WithComposes(bytes.NewReader(overrides)))
 	// todo: merge additional compose file
 	if err != nil {
 		return err
@@ -84,3 +93,64 @@ func getBundleImageMap() (map[string]bundle.Image, error) {
 	}
 	return result, nil
 }
+
+func parseOverrides() ([]byte, error) {
+	root := make(map[string]interface{})
+	if err := filepath.Walk(internal.ComposeOverridesDir, func(path string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !fi.IsDir() && fi.Size() > 0 {
+			bytes, err := ioutil.ReadFile(path)
+			if err != nil {
+				return err
+			}
+			rel, err := filepath.Rel(internal.ComposeOverridesDir, path)
+			if err != nil {
+				return err
+			}
+			splitPath := strings.Split(rel, "/")
+			if err := setValue(root, splitPath, string(bytes)); err != nil {
+				return err
+			}
+		}
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+	return yaml.Marshal(root)
+}
+
+func setValue(root map[string]interface{}, path []string, value string) error {
+	key, sub := path[0], path[1:]
+	if len(sub) == 0 {
+		converted, err := converterFor(key)(value)
+		if err != nil {
+			return err
+		}
+		root[key] = converted
+		return nil
+	}
+	subMap := make(map[string]interface{})
+	root[key] = subMap
+	return setValue(subMap, sub, value)
+}
+
+type valueConverter func(string) (interface{}, error)
+
+func stringConverter(v string) (interface{}, error) {
+	return v, nil
+}
+
+func intConverter(v string) (interface{}, error) {
+	return strconv.ParseInt(v, 10, 32)
+}
+
+func converterFor(key string) valueConverter {
+	switch key {
+	case "replicas":
+		return intConverter
+	default:
+		return stringConverter
+	}
+}

cmd/cnab-run/render.go

@@ -1,18 +1,25 @@
 package main
 
 import (
+	"bytes"
 	"fmt"
 	"os"
 
 	"github.com/docker/app/internal"
+	"github.com/docker/app/types"
+	"github.com/pkg/errors"
 
 	"github.com/docker/app/internal/formatter"
 	"github.com/docker/app/internal/packager"
 	"github.com/docker/app/render"
 )
 
 func renderAction(instanceName string) error {
-	app, err := packager.Extract("")
+	overrides, err := parseOverrides()
+	if err != nil {
+		return errors.Wrap(err, "unable to parse auto-parameter values")
+	}
+	app, err := packager.Extract("", types.WithComposes(bytes.NewReader(overrides)))
 	// todo: merge additional compose file
 	if err != nil {
 		return err

e2e/pushpull_test.go

@@ -6,6 +6,7 @@ import (
 	"net"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -125,6 +126,26 @@ func TestPushPullInstall(t *testing.T) {
 	})
 }
 
+func TestAutomaticParameters(t *testing.T) {
+	runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) {
+		cmd := info.configuredCmd
+		ref := info.registryAddress + "/test/push-pull"
+		cmd.Command = dockerCli.Command("app", "push", "--tag", ref, "--insecure-registries="+info.registryAddress, filepath.Join("testdata", "push-pull", "push-pull.dockerapp"))
+		icmd.RunCmd(cmd).Assert(t, icmd.Success)
+		cmd.Command = dockerCli.Command("app", "install", "--insecure-registries="+info.registryAddress, ref, "--name", t.Name())
+		icmd.RunCmd(cmd).Assert(t, icmd.Success)
+		cmd.Command = dockerCli.Command("--context=swarm-target-context", "service", "inspect", t.Name()+"_web", "-f", "{{.Spec.Mode.Replicated.Replicas}}")
+		replicasOut := icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined()
+		assert.Equal(t, strings.TrimSpace(replicasOut), "1")
+
+		cmd.Command = dockerCli.Command("app", "upgrade", t.Name(), "-s", "services.web.deploy.replicas=2")
+		icmd.RunCmd(cmd).Assert(t, icmd.Success)
+		cmd.Command = dockerCli.Command("--context=swarm-target-context", "service", "inspect", t.Name()+"_web", "-f", "{{.Spec.Mode.Replicated.Replicas}}")
+		replicasOut = icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined()
+		assert.Equal(t, strings.TrimSpace(replicasOut), "2")
+	})
+}
+
 func findAvailablePort() int {
 	rand.Seed(time.Now().UnixNano())
 	for {

internal/names.go

@@ -40,6 +40,8 @@ const (
 	CredentialRegistryName = Namespace + "registry-creds"
 	// CredentialRegistryPath is the name to the credential containing registry credentials
 	CredentialRegistryPath = "/cnab/app/registry-creds.json"
+	// ComposeOverridesDir is the path where automatic parameters store their value overrides
+	ComposeOverridesDir = "/cnab/app/overrides"
 
 	// ParameterOrchestratorName is the name of the parameter containing the orchestrator
 	ParameterOrchestratorName = Namespace + "orchestrator"

internal/packager/cnab.go

@@ -1,10 +1,17 @@
 package packager
 
 import (
+	"fmt"
+	"path"
+	"strings"
+
 	"github.com/deislabs/duffle/pkg/bundle"
 	"github.com/docker/app/internal"
 	"github.com/docker/app/internal/compose"
 	"github.com/docker/app/types"
+	"github.com/docker/cli/cli/compose/loader"
+	"github.com/imdario/mergo"
+	"github.com/pkg/errors"
 )
 
 // ToCNAB creates a CNAB bundle from an app package
@@ -86,6 +93,15 @@ func ToCNAB(app *types.App, invocationImageName string) (*bundle.Bundle, error)
 			DefaultValue: flatParameters[name],
 		}
 	}
+	autoParams, err := generateOverrideParameters(app.Composes())
+	if err != nil {
+		return nil, fmt.Errorf("unable to generate automatic parameters: %s", err)
+	}
+	for k, v := range autoParams {
+		if _, exist := parameters[k]; !exist {
+			parameters[k] = v
+		}
+	}
 	var maintainers []bundle.Maintainer
 	for _, m := range app.Metadata().Maintainers {
 		maintainers = append(maintainers, bundle.Maintainer{
@@ -156,3 +172,78 @@ func extractBundleImages(composeFiles [][]byte) (map[string]bundle.Image, error)
 	}
 	return bundleImages, nil
 }
+
+func generateOverrideParameters(composeFiles [][]byte) (map[string]bundle.ParameterDefinition, error) {
+
+	merged := make(map[string]interface{})
+	for _, composeFile := range composeFiles {
+		parsed, err := loader.ParseYAML(composeFile)
+		if err != nil {
+			return nil, err
+		}
+		if err := mergo.Merge(&merged, parsed, mergo.WithAppendSlice, mergo.WithOverride); err != nil {
+			return nil, err
+		}
+	}
+	servicesRaw, ok := merged["services"]
+	if !ok {
+		return nil, nil
+	}
+	services, ok := servicesRaw.(map[string]interface{})
+	if !ok {
+		return nil, errors.New("unrecognized services type")
+	}
+	defs := make(map[string]bundle.ParameterDefinition)
+	for serviceName, serviceValue := range services {
+		serviceDef, ok := serviceValue.(map[string]interface{})
+		if !ok {
+			return nil, fmt.Errorf("unerecognized type for service %q", serviceName)
+		}
+		addServiceOverrideParameters(serviceName, serviceDef, defs)
+	}
+	return defs, nil
+}
+
+func addServiceOverrideParameters(serviceName string, serviceDef map[string]interface{}, into map[string]bundle.ParameterDefinition) {
+	for _, p := range serviceParametersToGenerate {
+		pathParts := strings.Split(p, ".")
+		if !hasKey(serviceDef, pathParts...) {
+			dest := path.Join(internal.ComposeOverridesDir, "services", serviceName, strings.Join(pathParts, "/"))
+			name := "services." + serviceName + "." + p
+			into[name] = bundle.ParameterDefinition{
+				DataType: "string",
+				Destination: &bundle.Location{
+					Path: dest,
+				},
+				DefaultValue: "",
+			}
+		}
+	}
+}
+
+var serviceParametersToGenerate = []string{
+	"deploy.replicas",
+	"deploy.resources.limits.cpus",
+	"deploy.resources.limits.memory",
+	"deploy.resources.reservations.cpus",
+	"deploy.resources.reservations.memory",
+}
+
+func hasKey(source map[string]interface{}, path ...string) bool {
+	if len(path) == 0 {
+		return true
+	}
+	key, remaining := path[0], path[1:]
+	subRaw, ok := source[key]
+	if !ok {
+		return false
+	}
+	if len(remaining) == 0 {
+		return true
+	}
+	sub, ok := subRaw.(map[string]interface{})
+	if !ok {
+		return false
+	}
+	return hasKey(sub, remaining...)
+}

internal/packager/cnab_test.go

@@ -2,12 +2,15 @@ package packager
 
 import (
 	"encoding/json"
+	"path"
+	"strings"
 	"testing"
 
-	"gotest.tools/golden"
-
+	"github.com/deislabs/duffle/pkg/bundle"
+	"github.com/docker/app/internal"
 	"github.com/docker/app/types"
 	"gotest.tools/assert"
+	"gotest.tools/golden"
 )
 
 func TestToCNAB(t *testing.T) {
@@ -19,3 +22,54 @@ func TestToCNAB(t *testing.T) {
 	assert.NilError(t, err)
 	golden.Assert(t, string(actualJSON), "bundle-json.golden")
 }
+
+func TestCnabAutomaticParameters(t *testing.T) {
+	app, err := types.NewAppFromDefaultFiles("testdata/packages/auto-parameters.dockerapp")
+	assert.NilError(t, err)
+	actual, err := ToCNAB(app, "test-image")
+	assert.NilError(t, err)
+	checkOverrideParameter(t, actual, "services.nothing-specified.deploy.replicas")
+	checkOverrideParameter(t, actual, "services.nothing-specified.deploy.resources.limits.cpus")
+	checkOverrideParameter(t, actual, "services.nothing-specified.deploy.resources.limits.memory")
+	checkOverrideParameter(t, actual, "services.nothing-specified.deploy.resources.reservations.cpus")
+	checkOverrideParameter(t, actual, "services.nothing-specified.deploy.resources.reservations.memory")
+	checkNoParameter(t, actual, "services.replicas-fixed.deploy.replicas")
+	checkOverrideParameter(t, actual, "services.replicas-fixed.deploy.resources.limits.cpus")
+	checkOverrideParameter(t, actual, "services.replicas-fixed.deploy.resources.limits.memory")
+	checkOverrideParameter(t, actual, "services.replicas-fixed.deploy.resources.reservations.cpus")
+	checkOverrideParameter(t, actual, "services.replicas-fixed.deploy.resources.reservations.memory")
+	checkCustomParameter(t, actual, "services.parameter-names-used.deploy.replicas")
+	checkCustomParameter(t, actual, "services.parameter-names-used.deploy.resources.limits.cpus")
+	checkCustomParameter(t, actual, "services.parameter-names-used.deploy.resources.limits.memory")
+	checkCustomParameter(t, actual, "services.parameter-names-used.deploy.resources.reservations.cpus")
+	checkCustomParameter(t, actual, "services.parameter-names-used.deploy.resources.reservations.memory")
+}
+
+func checkOverrideParameter(t *testing.T, b *bundle.Bundle, parameterName string) {
+	t.Helper()
+	parameterDest := path.Join(internal.ComposeOverridesDir, strings.ReplaceAll(parameterName, ".", "/"))
+	param, ok := b.Parameters[parameterName]
+	if !ok {
+		t.Fatalf("parameter %q is not present", parameterName)
+	}
+	assert.Check(t, param.Destination != nil)
+	assert.Equal(t, param.Destination.Path, parameterDest)
+}
+
+func checkNoParameter(t *testing.T, b *bundle.Bundle, parameterName string) {
+	t.Helper()
+	_, ok := b.Parameters[parameterName]
+	if ok {
+		t.Fatalf("parameter %q is present", parameterName)
+	}
+}
+
+func checkCustomParameter(t *testing.T, b *bundle.Bundle, parameterName string) {
+	t.Helper()
+	param, ok := b.Parameters[parameterName]
+	if !ok {
+		t.Fatalf("parameter %q is not present", parameterName)
+	}
+	assert.Check(t, param.Destination != nil)
+	assert.Equal(t, param.Destination.Path, "")
+}