implement normalization as a yaml transformation (#589)

* implement normalization as a yaml transformation

Signed-off-by: Nicolas De Loof <[email protected]>

* introduce LoadModel for full compose file loading without go bindings

Signed-off-by: Nicolas De Loof <[email protected]>

---------

Signed-off-by: Nicolas De Loof <[email protected]>

Author: ndeloof

.gitignore

@@ -1,3 +1,4 @@
 ### IDEs ###
 .idea/*
 .vscode/*
+bin/

Makefile

@@ -16,7 +16,7 @@ IMAGE_PREFIX=composespec/conformance-tests-
 
 .PHONY: build
 build: ## Build command line
-	go build -o compose-spec cmd/main.go
+	go build -o bin/compose-spec cmd/main.go
 
 .PHONY: test
 test: ## Run tests

cli/options.go

@@ -379,7 +379,7 @@ var DefaultFileNames = []string{"compose.yaml", "compose.yml", "docker-compose.y
 // DefaultOverrideFileNames defines the Compose override file names for auto-discovery (in order of preference)
 var DefaultOverrideFileNames = []string{"compose.override.yml", "compose.override.yaml", "docker-compose.override.yml", "docker-compose.override.yaml"}
 
-func (o ProjectOptions) GetWorkingDir() (string, error) {
+func (o *ProjectOptions) GetWorkingDir() (string, error) {
 	if o.WorkingDir != "" {
 		return filepath.Abs(o.WorkingDir)
 	}
@@ -395,7 +395,7 @@ func (o ProjectOptions) GetWorkingDir() (string, error) {
 	return os.Getwd()
 }
 
-func (o ProjectOptions) GeConfigFiles() ([]types.ConfigFile, error) {
+func (o *ProjectOptions) GeConfigFiles() ([]types.ConfigFile, error) {
 	configPaths, err := o.getConfigPaths()
 	if err != nil {
 		return nil, err
@@ -427,37 +427,64 @@ func (o ProjectOptions) GeConfigFiles() ([]types.ConfigFile, error) {
 	return configs, err
 }
 
-// ProjectFromOptions load a compose project based on command line options
-func ProjectFromOptions(ctx context.Context, options *ProjectOptions) (*types.Project, error) {
-	configs, err := options.GeConfigFiles()
+// LoadProject loads compose file according to options and bind to types.Project go structs
+func (o *ProjectOptions) LoadProject(ctx context.Context) (*types.Project, error) {
+	configDetails, err := o.prepare()
 	if err != nil {
 		return nil, err
 	}
 
-	workingDir, err := options.GetWorkingDir()
+	project, err := loader.LoadWithContext(ctx, configDetails, o.loadOptions...)
 	if err != nil {
 		return nil, err
 	}
 
-	options.loadOptions = append(options.loadOptions,
-		withNamePrecedenceLoad(workingDir, options),
-		withConvertWindowsPaths(options),
-		withListeners(options))
+	for _, config := range configDetails.ConfigFiles {
+		project.ComposeFiles = append(project.ComposeFiles, config.Filename)
+	}
 
-	project, err := loader.LoadWithContext(ctx, types.ConfigDetails{
-		ConfigFiles: configs,
-		WorkingDir:  workingDir,
-		Environment: options.Environment,
-	}, options.loadOptions...)
+	return project, nil
+}
+
+// LoadModel loads compose file according to options and returns a raw (yaml tree) model
+func (o *ProjectOptions) LoadModel(ctx context.Context) (map[string]any, error) {
+	configDetails, err := o.prepare()
 	if err != nil {
 		return nil, err
 	}
 
-	for _, config := range configs {
-		project.ComposeFiles = append(project.ComposeFiles, config.Filename)
+	return loader.LoadModelWithContext(ctx, configDetails, o.loadOptions...)
+}
+
+// prepare converts ProjectOptions into loader's types.ConfigDetails and configures default load options
+func (o *ProjectOptions) prepare() (types.ConfigDetails, error) {
+	configs, err := o.GeConfigFiles()
+	if err != nil {
+		return types.ConfigDetails{}, err
 	}
 
-	return project, nil
+	workingDir, err := o.GetWorkingDir()
+	if err != nil {
+		return types.ConfigDetails{}, err
+	}
+
+	configDetails := types.ConfigDetails{
+		ConfigFiles: configs,
+		WorkingDir:  workingDir,
+		Environment: o.Environment,
+	}
+
+	o.loadOptions = append(o.loadOptions,
+		withNamePrecedenceLoad(workingDir, o),
+		withConvertWindowsPaths(o),
+		withListeners(o))
+	return configDetails, nil
+}
+
+// ProjectFromOptions load a compose project based on command line options
+// Deprecated: use ProjectOptions.LoadProject or ProjectOptions.LoadModel
+func ProjectFromOptions(ctx context.Context, options *ProjectOptions) (*types.Project, error) {
+	return options.LoadProject(ctx)
 }
 
 func withNamePrecedenceLoad(absWorkingDir string, options *ProjectOptions) func(*loader.Options) {

cmd/main.go

@@ -18,11 +18,13 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"os"
 
 	"github.com/compose-spec/compose-go/v2/cli"
+	"gopkg.in/yaml.v3"
 )
 
 func main() {
@@ -34,11 +36,13 @@ Usage: compose-spec [OPTIONS] COMPOSE_FILE [COMPOSE_OVERRIDE_FILE]`)
 	}
 
 	var skipInterpolation, skipResolvePaths, skipNormalization, skipConsistencyCheck bool
+	var format string
 
 	flag.BoolVar(&skipInterpolation, "no-interpolation", false, "Don't interpolate environment variables.")
 	flag.BoolVar(&skipResolvePaths, "no-path-resolution", false, "Don't resolve file paths.")
 	flag.BoolVar(&skipNormalization, "no-normalization", false, "Don't normalize compose model.")
 	flag.BoolVar(&skipConsistencyCheck, "no-consistency", false, "Don't check model consistency.")
+	flag.StringVar(&format, "format", "yaml", "Output format (yaml|json).")
 	flag.Parse()
 
 	wd, err := os.Getwd()
@@ -61,16 +65,29 @@ Usage: compose-spec [OPTIONS] COMPOSE_FILE [COMPOSE_OVERRIDE_FILE]`)
 		exitError("failed to configure project options", err)
 	}
 
-	project, err := cli.ProjectFromOptions(context.Background(), options)
+	model, err := options.LoadModel(context.Background())
 	if err != nil {
 		exitError("failed to load project", err)
 	}
 
-	yaml, err := project.MarshalYAML()
-	if err != nil {
-		exitError("failed to marshall project", err)
+	var raw []byte
+	switch format {
+	case "yaml":
+		raw, err = yaml.Marshal(model)
+		if err != nil {
+			exitError("failed to marshall project", err)
+		}
+	case "json":
+		raw, err = json.MarshalIndent(model, "", "  ")
+		if err != nil {
+			exitError("failed to marshall project", err)
+		}
+	default:
+		_ = fmt.Errorf("unsupported output format %s", format)
+		os.Exit(1)
 	}
-	fmt.Println(string(yaml))
+
+	fmt.Println(string(raw))
 }
 
 func exitError(message string, err error) {

loader/loader.go

@@ -285,27 +285,29 @@ func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types.
 	return LoadWithContext(context.Background(), configDetails, options...)
 }
 
-// LoadWithContext reads a ConfigDetails and returns a fully loaded configuration
+// LoadWithContext reads a ConfigDetails and returns a fully loaded configuration as a compose-go Project
 func LoadWithContext(ctx context.Context, configDetails types.ConfigDetails, options ...func(*Options)) (*types.Project, error) {
-	if len(configDetails.ConfigFiles) < 1 {
-		return nil, errors.New("No files specified")
+	opts := toOptions(&configDetails, options)
+	dict, err := loadModelWithContext(ctx, &configDetails, opts)
+	if err != nil {
+		return nil, err
 	}
+	return modelToProject(dict, opts, configDetails)
+}
 
-	opts := &Options{
-		Interpolate: &interp.Options{
-			Substitute:      template.Substitute,
-			LookupValue:     configDetails.LookupEnv,
-			TypeCastMapping: interpolateTypeCastMapping,
-		},
-		ResolvePaths: true,
-	}
+// LoadModelWithContext reads a ConfigDetails and returns a fully loaded configuration as a yaml dictionary
+func LoadModelWithContext(ctx context.Context, configDetails types.ConfigDetails, options ...func(*Options)) (map[string]any, error) {
+	opts := toOptions(&configDetails, options)
+	return loadModelWithContext(ctx, &configDetails, opts)
+}
 
-	for _, op := range options {
-		op(opts)
+// LoadModelWithContext reads a ConfigDetails and returns a fully loaded configuration as a yaml dictionary
+func loadModelWithContext(ctx context.Context, configDetails *types.ConfigDetails, opts *Options) (map[string]any, error) {
+	if len(configDetails.ConfigFiles) < 1 {
+		return nil, errors.New("No files specified")
 	}
-	opts.ResourceLoaders = append(opts.ResourceLoaders, localResourceLoader{configDetails.WorkingDir})
 
-	err := projectName(configDetails, opts)
+	err := projectName(*configDetails, opts)
 	if err != nil {
 		return nil, err
 	}
@@ -318,7 +320,24 @@ func LoadWithContext(ctx context.Context, configDetails types.ConfigDetails, opt
 		configDetails.Environment[consts.ComposeProjectName] = opts.projectName
 	}
 
-	return load(ctx, configDetails, opts, nil)
+	return load(ctx, *configDetails, opts, nil)
+}
+
+func toOptions(configDetails *types.ConfigDetails, options []func(*Options)) *Options {
+	opts := &Options{
+		Interpolate: &interp.Options{
+			Substitute:      template.Substitute,
+			LookupValue:     configDetails.LookupEnv,
+			TypeCastMapping: interpolateTypeCastMapping,
+		},
+		ResolvePaths: true,
+	}
+
+	for _, op := range options {
+		op(opts)
+	}
+	opts.ResourceLoaders = append(opts.ResourceLoaders, localResourceLoader{configDetails.WorkingDir})
+	return opts
 }
 
 func loadYamlModel(ctx context.Context, config types.ConfigDetails, opts *Options, ct *cycleTracker, included []string) (map[string]interface{}, error) {
@@ -458,7 +477,7 @@ func loadYamlModel(ctx context.Context, config types.ConfigDetails, opts *Option
 	return dict, nil
 }
 
-func load(ctx context.Context, configDetails types.ConfigDetails, opts *Options, loaded []string) (*types.Project, error) {
+func load(ctx context.Context, configDetails types.ConfigDetails, opts *Options, loaded []string) (map[string]interface{}, error) {
 	mainFile := configDetails.ConfigFiles[0].Filename
 	for _, f := range loaded {
 		if f == mainFile {
@@ -481,13 +500,26 @@ func load(ctx context.Context, configDetails types.ConfigDetails, opts *Options,
 		return nil, errors.New("project name must not be empty")
 	}
 
+	if !opts.SkipNormalization {
+		dict, err = Normalize(dict, configDetails.Environment)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return dict, nil
+}
+
+// modelToProject binds a canonical yaml dict into compose-go structs
+func modelToProject(dict map[string]interface{}, opts *Options, configDetails types.ConfigDetails) (*types.Project, error) {
 	project := &types.Project{
 		Name:        opts.projectName,
 		WorkingDir:  configDetails.WorkingDir,
 		Environment: configDetails.Environment,
 	}
 	delete(dict, "name") // project name set by yaml must be identified by caller as opts.projectName
 
+	var err error
 	dict, err = processExtensions(dict, tree.NewPath(), opts.KnownExtensions)
 	if err != nil {
 		return nil, err
@@ -498,13 +530,6 @@ func load(ctx context.Context, configDetails types.ConfigDetails, opts *Options,
 		return nil, err
 	}
 
-	if !opts.SkipNormalization {
-		err := Normalize(project)
-		if err != nil {
-			return nil, err
-		}
-	}
-
 	if opts.ConvertWindowsPaths {
 		for i, service := range project.Services {
 			for j, volume := range service.Volumes {
@@ -531,7 +556,6 @@ func load(ctx context.Context, configDetails types.ConfigDetails, opts *Options,
 			return nil, err
 		}
 	}
-
 	return project, nil
 }
 

loader/loader_test.go

@@ -2325,7 +2325,7 @@ services:
     container_name: ${COMPOSE_PROJECT_NAME}-web
 `
 		configDetails := buildConfigDetails(yaml, map[string]string{"COMPOSE_PROJECT_NAME": "env-var"})
-		actual, err := Load(configDetails)
+		actual, err := LoadWithContext(context.TODO(), configDetails)
 		assert.NilError(t, err)
 		svc, err := actual.GetService("web")
 		assert.NilError(t, err)