diff --git a/cmd/cnab-run/inspect.go b/cmd/cnab-run/inspect.go index 8d55835ef..9503aeec2 100644 --- a/cmd/cnab-run/inspect.go +++ b/cmd/cnab-run/inspect.go @@ -1,14 +1,21 @@ package main import ( + "bytes" "os" appinspect "github.com/docker/app/internal/inspect" "github.com/docker/app/internal/packager" + "github.com/docker/app/types" + "github.com/pkg/errors" ) func inspectAction(instanceName string) error { - app, err := packager.Extract("") + overrides, err := parseOverrides() + if err != nil { + return errors.Wrap(err, "unable to parse auto-parameter values") + } + app, err := packager.Extract("", types.WithComposes(bytes.NewReader(overrides))) // todo: merge additional compose file if err != nil { return err diff --git a/cmd/cnab-run/install.go b/cmd/cnab-run/install.go index 72adb5764..8e5e38a3c 100644 --- a/cmd/cnab-run/install.go +++ b/cmd/cnab-run/install.go @@ -1,21 +1,26 @@ package main import ( + "bytes" "encoding/json" "io/ioutil" "os" + "path/filepath" "strconv" + "strings" "github.com/deislabs/duffle/pkg/bundle" "github.com/docker/app/internal" "github.com/docker/app/internal/packager" "github.com/docker/app/render" + "github.com/docker/app/types" "github.com/docker/cli/cli/command" "github.com/docker/cli/cli/command/stack" "github.com/docker/cli/cli/command/stack/options" "github.com/docker/cli/cli/command/stack/swarm" "github.com/pkg/errors" "github.com/spf13/pflag" + yaml "gopkg.in/yaml.v2" ) const ( @@ -29,7 +34,11 @@ func installAction(instanceName string) error { if err != nil { return errors.Wrap(err, "unable to restore docker context") } - app, err := packager.Extract("") + overrides, err := parseOverrides() + if err != nil { + return errors.Wrap(err, "unable to parse auto-parameter values") + } + app, err := packager.Extract("", types.WithComposes(bytes.NewReader(overrides))) // todo: merge additional compose file if err != nil { return err @@ -84,3 +93,64 @@ func getBundleImageMap() (map[string]bundle.Image, error) { } return result, nil } + +func parseOverrides() ([]byte, error) { + root := make(map[string]interface{}) + if err := filepath.Walk(internal.ComposeOverridesDir, func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + if !fi.IsDir() && fi.Size() > 0 { + bytes, err := ioutil.ReadFile(path) + if err != nil { + return err + } + rel, err := filepath.Rel(internal.ComposeOverridesDir, path) + if err != nil { + return err + } + splitPath := strings.Split(rel, "/") + if err := setValue(root, splitPath, string(bytes)); err != nil { + return err + } + } + return nil + }); err != nil { + return nil, err + } + return yaml.Marshal(root) +} + +func setValue(root map[string]interface{}, path []string, value string) error { + key, sub := path[0], path[1:] + if len(sub) == 0 { + converted, err := converterFor(key)(value) + if err != nil { + return err + } + root[key] = converted + return nil + } + subMap := make(map[string]interface{}) + root[key] = subMap + return setValue(subMap, sub, value) +} + +type valueConverter func(string) (interface{}, error) + +func stringConverter(v string) (interface{}, error) { + return v, nil +} + +func intConverter(v string) (interface{}, error) { + return strconv.ParseInt(v, 10, 32) +} + +func converterFor(key string) valueConverter { + switch key { + case "replicas": + return intConverter + default: + return stringConverter + } +} diff --git a/cmd/cnab-run/render.go b/cmd/cnab-run/render.go index 91e486564..9f140fbe9 100644 --- a/cmd/cnab-run/render.go +++ b/cmd/cnab-run/render.go @@ -1,10 +1,13 @@ package main import ( + "bytes" "fmt" "os" "github.com/docker/app/internal" + "github.com/docker/app/types" + "github.com/pkg/errors" "github.com/docker/app/internal/formatter" "github.com/docker/app/internal/packager" @@ -12,7 +15,11 @@ import ( ) func renderAction(instanceName string) error { - app, err := packager.Extract("") + overrides, err := parseOverrides() + if err != nil { + return errors.Wrap(err, "unable to parse auto-parameter values") + } + app, err := packager.Extract("", types.WithComposes(bytes.NewReader(overrides))) // todo: merge additional compose file if err != nil { return err diff --git a/e2e/pushpull_test.go b/e2e/pushpull_test.go index be22f722b..4ecced716 100644 --- a/e2e/pushpull_test.go +++ b/e2e/pushpull_test.go @@ -6,6 +6,7 @@ import ( "net" "path/filepath" "strconv" + "strings" "testing" "time" @@ -125,6 +126,26 @@ func TestPushPullInstall(t *testing.T) { }) } +func TestAutomaticParameters(t *testing.T) { + runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + cmd := info.configuredCmd + ref := info.registryAddress + "/test/push-pull" + cmd.Command = dockerCli.Command("app", "push", "--tag", ref, "--insecure-registries="+info.registryAddress, filepath.Join("testdata", "push-pull", "push-pull.dockerapp")) + icmd.RunCmd(cmd).Assert(t, icmd.Success) + cmd.Command = dockerCli.Command("app", "install", "--insecure-registries="+info.registryAddress, ref, "--name", t.Name()) + icmd.RunCmd(cmd).Assert(t, icmd.Success) + cmd.Command = dockerCli.Command("--context=swarm-target-context", "service", "inspect", t.Name()+"_web", "-f", "{{.Spec.Mode.Replicated.Replicas}}") + replicasOut := icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined() + assert.Equal(t, strings.TrimSpace(replicasOut), "1") + + cmd.Command = dockerCli.Command("app", "upgrade", t.Name(), "-s", "services.web.deploy.replicas=2") + icmd.RunCmd(cmd).Assert(t, icmd.Success) + cmd.Command = dockerCli.Command("--context=swarm-target-context", "service", "inspect", t.Name()+"_web", "-f", "{{.Spec.Mode.Replicated.Replicas}}") + replicasOut = icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined() + assert.Equal(t, strings.TrimSpace(replicasOut), "2") + }) +} + func findAvailablePort() int { rand.Seed(time.Now().UnixNano()) for { diff --git a/internal/names.go b/internal/names.go index 030b3d2eb..fa003cd58 100644 --- a/internal/names.go +++ b/internal/names.go @@ -40,6 +40,8 @@ const ( CredentialRegistryName = Namespace + "registry-creds" // CredentialRegistryPath is the name to the credential containing registry credentials CredentialRegistryPath = "/cnab/app/registry-creds.json" + // ComposeOverridesDir is the path where automatic parameters store their value overrides + ComposeOverridesDir = "/cnab/app/overrides" // ParameterOrchestratorName is the name of the parameter containing the orchestrator ParameterOrchestratorName = Namespace + "orchestrator" diff --git a/internal/packager/cnab.go b/internal/packager/cnab.go index a1675adbb..21441619c 100644 --- a/internal/packager/cnab.go +++ b/internal/packager/cnab.go @@ -1,10 +1,17 @@ package packager import ( + "fmt" + "path" + "strings" + "github.com/deislabs/duffle/pkg/bundle" "github.com/docker/app/internal" "github.com/docker/app/internal/compose" "github.com/docker/app/types" + "github.com/docker/cli/cli/compose/loader" + "github.com/imdario/mergo" + "github.com/pkg/errors" ) // ToCNAB creates a CNAB bundle from an app package @@ -86,6 +93,15 @@ func ToCNAB(app *types.App, invocationImageName string) (*bundle.Bundle, error) DefaultValue: flatParameters[name], } } + autoParams, err := generateOverrideParameters(app.Composes()) + if err != nil { + return nil, fmt.Errorf("unable to generate automatic parameters: %s", err) + } + for k, v := range autoParams { + if _, exist := parameters[k]; !exist { + parameters[k] = v + } + } var maintainers []bundle.Maintainer for _, m := range app.Metadata().Maintainers { maintainers = append(maintainers, bundle.Maintainer{ @@ -156,3 +172,78 @@ func extractBundleImages(composeFiles [][]byte) (map[string]bundle.Image, error) } return bundleImages, nil } + +func generateOverrideParameters(composeFiles [][]byte) (map[string]bundle.ParameterDefinition, error) { + + merged := make(map[string]interface{}) + for _, composeFile := range composeFiles { + parsed, err := loader.ParseYAML(composeFile) + if err != nil { + return nil, err + } + if err := mergo.Merge(&merged, parsed, mergo.WithAppendSlice, mergo.WithOverride); err != nil { + return nil, err + } + } + servicesRaw, ok := merged["services"] + if !ok { + return nil, nil + } + services, ok := servicesRaw.(map[string]interface{}) + if !ok { + return nil, errors.New("unrecognized services type") + } + defs := make(map[string]bundle.ParameterDefinition) + for serviceName, serviceValue := range services { + serviceDef, ok := serviceValue.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("unerecognized type for service %q", serviceName) + } + addServiceOverrideParameters(serviceName, serviceDef, defs) + } + return defs, nil +} + +func addServiceOverrideParameters(serviceName string, serviceDef map[string]interface{}, into map[string]bundle.ParameterDefinition) { + for _, p := range serviceParametersToGenerate { + pathParts := strings.Split(p, ".") + if !hasKey(serviceDef, pathParts...) { + dest := path.Join(internal.ComposeOverridesDir, "services", serviceName, strings.Join(pathParts, "/")) + name := "services." + serviceName + "." + p + into[name] = bundle.ParameterDefinition{ + DataType: "string", + Destination: &bundle.Location{ + Path: dest, + }, + DefaultValue: "", + } + } + } +} + +var serviceParametersToGenerate = []string{ + "deploy.replicas", + "deploy.resources.limits.cpus", + "deploy.resources.limits.memory", + "deploy.resources.reservations.cpus", + "deploy.resources.reservations.memory", +} + +func hasKey(source map[string]interface{}, path ...string) bool { + if len(path) == 0 { + return true + } + key, remaining := path[0], path[1:] + subRaw, ok := source[key] + if !ok { + return false + } + if len(remaining) == 0 { + return true + } + sub, ok := subRaw.(map[string]interface{}) + if !ok { + return false + } + return hasKey(sub, remaining...) +} diff --git a/internal/packager/cnab_test.go b/internal/packager/cnab_test.go index 519837a80..ccbf1a2cc 100644 --- a/internal/packager/cnab_test.go +++ b/internal/packager/cnab_test.go @@ -2,12 +2,15 @@ package packager import ( "encoding/json" + "path" + "strings" "testing" - "gotest.tools/golden" - + "github.com/deislabs/duffle/pkg/bundle" + "github.com/docker/app/internal" "github.com/docker/app/types" "gotest.tools/assert" + "gotest.tools/golden" ) func TestToCNAB(t *testing.T) { @@ -19,3 +22,54 @@ func TestToCNAB(t *testing.T) { assert.NilError(t, err) golden.Assert(t, string(actualJSON), "bundle-json.golden") } + +func TestCnabAutomaticParameters(t *testing.T) { + app, err := types.NewAppFromDefaultFiles("testdata/packages/auto-parameters.dockerapp") + assert.NilError(t, err) + actual, err := ToCNAB(app, "test-image") + assert.NilError(t, err) + checkOverrideParameter(t, actual, "services.nothing-specified.deploy.replicas") + checkOverrideParameter(t, actual, "services.nothing-specified.deploy.resources.limits.cpus") + checkOverrideParameter(t, actual, "services.nothing-specified.deploy.resources.limits.memory") + checkOverrideParameter(t, actual, "services.nothing-specified.deploy.resources.reservations.cpus") + checkOverrideParameter(t, actual, "services.nothing-specified.deploy.resources.reservations.memory") + checkNoParameter(t, actual, "services.replicas-fixed.deploy.replicas") + checkOverrideParameter(t, actual, "services.replicas-fixed.deploy.resources.limits.cpus") + checkOverrideParameter(t, actual, "services.replicas-fixed.deploy.resources.limits.memory") + checkOverrideParameter(t, actual, "services.replicas-fixed.deploy.resources.reservations.cpus") + checkOverrideParameter(t, actual, "services.replicas-fixed.deploy.resources.reservations.memory") + checkCustomParameter(t, actual, "services.parameter-names-used.deploy.replicas") + checkCustomParameter(t, actual, "services.parameter-names-used.deploy.resources.limits.cpus") + checkCustomParameter(t, actual, "services.parameter-names-used.deploy.resources.limits.memory") + checkCustomParameter(t, actual, "services.parameter-names-used.deploy.resources.reservations.cpus") + checkCustomParameter(t, actual, "services.parameter-names-used.deploy.resources.reservations.memory") +} + +func checkOverrideParameter(t *testing.T, b *bundle.Bundle, parameterName string) { + t.Helper() + parameterDest := path.Join(internal.ComposeOverridesDir, strings.ReplaceAll(parameterName, ".", "/")) + param, ok := b.Parameters[parameterName] + if !ok { + t.Fatalf("parameter %q is not present", parameterName) + } + assert.Check(t, param.Destination != nil) + assert.Equal(t, param.Destination.Path, parameterDest) +} + +func checkNoParameter(t *testing.T, b *bundle.Bundle, parameterName string) { + t.Helper() + _, ok := b.Parameters[parameterName] + if ok { + t.Fatalf("parameter %q is present", parameterName) + } +} + +func checkCustomParameter(t *testing.T, b *bundle.Bundle, parameterName string) { + t.Helper() + param, ok := b.Parameters[parameterName] + if !ok { + t.Fatalf("parameter %q is not present", parameterName) + } + assert.Check(t, param.Destination != nil) + assert.Equal(t, param.Destination.Path, "") +} diff --git a/internal/packager/packing.go b/internal/packager/packing.go index ac0d7a73d..e3b9c8d6c 100644 --- a/internal/packager/packing.go +++ b/internal/packager/packing.go @@ -8,6 +8,8 @@ import ( "os" "path/filepath" + "github.com/docker/app/internal/yaml" + "github.com/docker/app/internal" "github.com/docker/app/types" "github.com/docker/docker/pkg/archive" @@ -78,9 +80,21 @@ func PackInvocationImageContext(app *types.App, target io.Writer) error { return errors.Wrapf(err, "failed to add attachment %q to the invocation image build context", prefix+attachment.Path()) } } + // extract compose version, and use the same in overrides + var v composeVersion + if err := yaml.Unmarshal(app.Composes()[0], &v); err != nil { + return err + } + if err := tarAddBytes(tarout, "overrides/version", []byte(v.Version)); err != nil { + return err + } return nil } +type composeVersion struct { + Version string `yaml:"version"` +} + // Pack packs the app as a single file func Pack(appname string, target io.Writer) error { tarout := tar.NewWriter(target) diff --git a/internal/packager/packing_test.go b/internal/packager/packing_test.go index 92b5730bf..68bcaee79 100644 --- a/internal/packager/packing_test.go +++ b/internal/packager/packing_test.go @@ -5,6 +5,7 @@ import ( "bytes" "fmt" "io" + "path" "strings" "testing" @@ -26,6 +27,7 @@ func TestPackInvocationImageContext(t *testing.T) { "packing.dockerapp/config.cfg": true, "packing.dockerapp/nesteddir/config2.cfg": true, "packing.dockerapp/nesteddir/nested2/nested3/config3.cfg": true, + path.Join("overrides/version"): true, })) } diff --git a/internal/packager/testdata/bundle-json.golden b/internal/packager/testdata/bundle-json.golden index cfa1a2fab..6f079534d 100644 --- a/internal/packager/testdata/bundle-json.golden +++ b/internal/packager/testdata/bundle-json.golden @@ -114,6 +114,114 @@ "env": "DOCKER_SHARE_REGISTRY_CREDS" } }, + "services.app-watcher.deploy.replicas": { + "type": "string", + "destination": { + "path": "/cnab/app/overrides/services/app-watcher/deploy/replicas" + } + }, + "services.app-watcher.deploy.resources.limits.cpus": { + "type": "string", + "destination": { + "path": "/cnab/app/overrides/services/app-watcher/deploy/resources/limits/cpus" + } + }, + "services.app-watcher.deploy.resources.limits.memory": { + "type": "string", + "destination": { + "path": "/cnab/app/overrides/services/app-watcher/deploy/resources/limits/memory" + } + }, + "services.app-watcher.deploy.resources.reservations.cpus": { + "type": "string", + "destination": { + "path": "/cnab/app/overrides/services/app-watcher/deploy/resources/reservations/cpus" + } + }, + "services.app-watcher.deploy.resources.reservations.memory": { + "type": "string", + "destination": { + "path": "/cnab/app/overrides/services/app-watcher/deploy/resources/reservations/memory" + } + }, + "services.debug.deploy.replicas": { + "type": "string", + "destination": { + "path": "/cnab/app/overrides/services/debug/deploy/replicas" + } + }, + "services.debug.deploy.resources.limits.cpus": { + "type": "string", + "destination": { + "path": "/cnab/app/overrides/services/debug/deploy/resources/limits/cpus" + } + }, + "services.debug.deploy.resources.reservations.cpus": { + "type": "string", + "destination": { + "path": "/cnab/app/overrides/services/debug/deploy/resources/reservations/cpus" + } + }, + "services.debug.deploy.resources.reservations.memory": { + "type": "string", + "destination": { + "path": "/cnab/app/overrides/services/debug/deploy/resources/reservations/memory" + } + }, + "services.front.deploy.resources.limits.cpus": { + "type": "string", + "destination": { + "path": "/cnab/app/overrides/services/front/deploy/resources/limits/cpus" + } + }, + "services.front.deploy.resources.limits.memory": { + "type": "string", + "destination": { + "path": "/cnab/app/overrides/services/front/deploy/resources/limits/memory" + } + }, + "services.front.deploy.resources.reservations.cpus": { + "type": "string", + "destination": { + "path": "/cnab/app/overrides/services/front/deploy/resources/reservations/cpus" + } + }, + "services.front.deploy.resources.reservations.memory": { + "type": "string", + "destination": { + "path": "/cnab/app/overrides/services/front/deploy/resources/reservations/memory" + } + }, + "services.monitor.deploy.replicas": { + "type": "string", + "destination": { + "path": "/cnab/app/overrides/services/monitor/deploy/replicas" + } + }, + "services.monitor.deploy.resources.limits.cpus": { + "type": "string", + "destination": { + "path": "/cnab/app/overrides/services/monitor/deploy/resources/limits/cpus" + } + }, + "services.monitor.deploy.resources.limits.memory": { + "type": "string", + "destination": { + "path": "/cnab/app/overrides/services/monitor/deploy/resources/limits/memory" + } + }, + "services.monitor.deploy.resources.reservations.cpus": { + "type": "string", + "destination": { + "path": "/cnab/app/overrides/services/monitor/deploy/resources/reservations/cpus" + } + }, + "services.monitor.deploy.resources.reservations.memory": { + "type": "string", + "destination": { + "path": "/cnab/app/overrides/services/monitor/deploy/resources/reservations/memory" + } + }, "watcher.cmd": { "type": "string", "defaultValue": "foo", diff --git a/internal/packager/testdata/packages/auto-parameters.dockerapp/docker-compose.yml b/internal/packager/testdata/packages/auto-parameters.dockerapp/docker-compose.yml new file mode 100644 index 000000000..c7a18129a --- /dev/null +++ b/internal/packager/testdata/packages/auto-parameters.dockerapp/docker-compose.yml @@ -0,0 +1,10 @@ +version: "3.7" +services: + nothing-specified: + image: nginx + replicas-fixed: + image: nginx + deploy: + replicas: 1 + parameter-names-used: + image: nginx \ No newline at end of file diff --git a/internal/packager/testdata/packages/auto-parameters.dockerapp/metadata.yml b/internal/packager/testdata/packages/auto-parameters.dockerapp/metadata.yml new file mode 100644 index 000000000..5cdfc34a4 --- /dev/null +++ b/internal/packager/testdata/packages/auto-parameters.dockerapp/metadata.yml @@ -0,0 +1,8 @@ +version: 0.1.0 +name: auto-parameters +description: "test-package-for-auto-params-generation" +maintainers: + - name: dev1 + email: dev1@example.com + - name: dev2 + email: dev2@example.com diff --git a/internal/packager/testdata/packages/auto-parameters.dockerapp/parameters.yml b/internal/packager/testdata/packages/auto-parameters.dockerapp/parameters.yml new file mode 100644 index 000000000..3f4c7a0ec --- /dev/null +++ b/internal/packager/testdata/packages/auto-parameters.dockerapp/parameters.yml @@ -0,0 +1,11 @@ +services: + parameter-names-used: + deploy: + replicas: 2 + resources: + limits: + cpus: '0.50' + memory: 50M + reservations: + cpus: '0.25' + memory: 20M \ No newline at end of file