Support for multiple compose files (#603)

Fixes #509 

In #500, we were concerned that the presence of both a `compose.yaml` and ` docker-compose.yaml` file could be confusing, so we decided to return an error and exit early. However, `compose-go` _does_ log out a warning in this scenario, as does `docker compose`. We've changed our mind and decided to aim for closer parity with docker here by reverting to logging the warning and continuing with `compose.yaml`.

While implementing this, in an effort to bring us to closer parity with docker compose, we decided to go ahead and add support for passing `-f` multiple times—adding support for multiple compose files.

- [x] Add support for multiple compose files with `-f`.
- [x] Modify `compose.LoadCompose` to leverage the `compose-go` library to find default compose file paths if one isn't passed explicitly.
- [x] Avoid returning an error if multiple default compose files are found. `compose-go` will always pick `compose.yaml` over `docker-compose.yaml`. It will also log a warning message explaining this to stderr.
- [x] Remove `toomany` test case since there is nothing meaningful to test anymore.
- [x] Add a simple test at the command level for something like `defang version` to make sure it doesn't try to load a compose file during intialization. In time, we should add command-level tests for everything with a mock server.
- [x] lazily verify compose path, to avoid requiring it for commands like `defang version` which do not use it.

Author: jordanstephens

src/cmd/cli/command/commands.go

@@ -136,7 +136,7 @@ func SetupCommands(version string) {
 	RootCmd.PersistentFlags().BoolVarP(&nonInteractive, "non-interactive", "T", !hasTty, "disable interactive prompts / no TTY")
 	RootCmd.PersistentFlags().StringP("cwd", "C", "", "change directory before running the command")
 	_ = RootCmd.MarkPersistentFlagDirname("cwd")
-	RootCmd.PersistentFlags().StringP("file", "f", "", `compose file path`)
+	RootCmd.PersistentFlags().StringArrayP("file", "f", []string{}, `compose file path`)
 	_ = RootCmd.MarkPersistentFlagFilename("file", "yml", "yaml")
 
 	// Bootstrap command
@@ -183,6 +183,7 @@ func SetupCommands(version string) {
 	// Config Command (was: secrets)
 	configSetCmd.Flags().BoolP("name", "n", false, "name of the config (backwards compat)")
 	_ = configSetCmd.Flags().MarkHidden("name")
+
 	configCmd.AddCommand(configSetCmd)
 
 	configDeleteCmd.Flags().BoolP("name", "n", false, "name of the config(s) (backwards compat)")
@@ -320,9 +321,7 @@ var RootCmd = &cobra.Command{
 				return err
 			}
 		}
-
-		composeFilePath, _ := cmd.Flags().GetString("file")
-		loader := compose.Loader{ComposeFilePath: composeFilePath}
+		loader := configureLoader(cmd)
 		client = cli.NewClient(cmd.Context(), cluster, provider, loader)
 
 		if v, err := client.GetVersions(cmd.Context()); err == nil {
@@ -583,7 +582,11 @@ var generateCmd = &cobra.Command{
 		}
 
 		// Load the project and check for empty environment variables
-		loader := compose.Loader{ComposeFilePath: filepath.Join(prompt.Folder, "compose.yaml")}
+		loaderOptions := compose.LoaderOptions{
+			WorkingDir:  prompt.Folder,
+			ConfigPaths: []string{filepath.Join(prompt.Folder, "compose.yaml")},
+		}
+		loader := compose.NewLoaderWithOptions(loaderOptions)
 		project, _ := loader.LoadCompose(cmd.Context())
 
 		var envInstructions []string
@@ -1268,6 +1271,24 @@ var tosCmd = &cobra.Command{
 	},
 }
 
+func configureLoader(cmd *cobra.Command) compose.Loader {
+	f := cmd.Flags()
+	o := compose.LoaderOptions{}
+	var err error
+
+	o.ConfigPaths, err = f.GetStringArray("file")
+	if err != nil {
+		panic(err)
+	}
+
+	o.WorkingDir, err = f.GetString("cwd")
+	if err != nil {
+		panic(err)
+	}
+
+	return compose.NewLoaderWithOptions(o)
+}
+
 func awsInEnv() bool {
 	return os.Getenv("AWS_PROFILE") != "" || os.Getenv("AWS_ACCESS_KEY_ID") != "" || os.Getenv("AWS_SECRET_ACCESS_KEY") != ""
 }

src/cmd/cli/command/commands_test.go

@@ -0,0 +1,20 @@
+package command
+
+import (
+	"context"
+	"testing"
+)
+
+func TestVersion(t *testing.T) {
+	err := testCommand([]string{"version"})
+	if err != nil {
+		t.Fatalf("Version() failed: %v", err)
+	}
+}
+
+func testCommand(args []string) error {
+	ctx := context.Background()
+	SetupCommands("test")
+	RootCmd.SetArgs(args)
+	return RootCmd.ExecuteContext(ctx)
+}

src/pkg/cli/compose/compose_test.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"os"
-	"strings"
 	"testing"
 
 	"github.com/DefangLabs/defang/src/pkg/term"
@@ -14,7 +13,7 @@ func TestLoadCompose(t *testing.T) {
 	term.SetDebug(testing.Verbose())
 
 	t.Run("no project name defaults to parent directory name", func(t *testing.T) {
-		loader := Loader{"../../../tests/noprojname/compose.yaml"}
+		loader := NewLoaderWithPath("../../../tests/noprojname/compose.yaml")
 		p, err := loader.LoadCompose(context.Background())
 		if err != nil {
 			t.Fatalf("LoadCompose() failed: %v", err)
@@ -25,7 +24,7 @@ func TestLoadCompose(t *testing.T) {
 	})
 
 	t.Run("no project name defaults to fancy parent directory name", func(t *testing.T) {
-		loader := Loader{"../../../tests/Fancy-Proj_Dir/compose.yaml"}
+		loader := NewLoaderWithPath("../../../tests/Fancy-Proj_Dir/compose.yaml")
 		p, err := loader.LoadCompose(context.Background())
 		if err != nil {
 			t.Fatalf("LoadCompose() failed: %v", err)
@@ -36,7 +35,7 @@ func TestLoadCompose(t *testing.T) {
 	})
 
 	t.Run("use project name in compose file", func(t *testing.T) {
-		loader := Loader{"../../../tests/testproj/compose.yaml"}
+		loader := NewLoaderWithPath("../../../tests/testproj/compose.yaml")
 		p, err := loader.LoadCompose(context.Background())
 		if err != nil {
 			t.Fatalf("LoadCompose() failed: %v", err)
@@ -48,7 +47,7 @@ func TestLoadCompose(t *testing.T) {
 
 	t.Run("COMPOSE_PROJECT_NAME env var should override project name", func(t *testing.T) {
 		t.Setenv("COMPOSE_PROJECT_NAME", "overridename")
-		loader := Loader{"../../../tests/testproj/compose.yaml"}
+		loader := NewLoaderWithPath("../../../tests/testproj/compose.yaml")
 		p, err := loader.LoadCompose(context.Background())
 		if err != nil {
 			t.Fatalf("LoadCompose() failed: %v", err)
@@ -59,7 +58,7 @@ func TestLoadCompose(t *testing.T) {
 	})
 
 	t.Run("use project name should not be overriden by tenantID", func(t *testing.T) {
-		loader := Loader{"../../../tests/testproj/compose.yaml"}
+		loader := NewLoaderWithPath("../../../tests/testproj/compose.yaml")
 		p, err := loader.LoadCompose(context.Background())
 		if err != nil {
 			t.Fatalf("LoadCompose() failed: %v", err)
@@ -88,7 +87,7 @@ func TestLoadCompose(t *testing.T) {
 		t.Cleanup(teardown)
 
 		// execute test
-		loader := Loader{}
+		loader := NewLoaderWithPath("")
 		p, err := loader.LoadCompose(context.Background())
 		if err != nil {
 			t.Fatalf("LoadCompose() failed: %v", err)
@@ -99,7 +98,7 @@ func TestLoadCompose(t *testing.T) {
 	})
 
 	t.Run("load alternative compose file", func(t *testing.T) {
-		loader := Loader{"../../../tests/alttestproj/altcomp.yaml"}
+		loader := NewLoaderWithPath("../../../tests/alttestproj/altcomp.yaml")
 		p, err := loader.LoadCompose(context.Background())
 		if err != nil {
 			t.Fatalf("LoadCompose() failed: %v", err)
@@ -120,7 +119,7 @@ func TestComposeGoNoDoubleWarningLog(t *testing.T) {
 	var warnings bytes.Buffer
 	term.DefaultTerm = term.NewTerm(&warnings, &warnings)
 
-	loader := Loader{"../../../tests/compose-go-warn/compose.yaml"}
+	loader := NewLoaderWithPath("../../../tests/compose-go-warn/compose.yaml")
 	_, err := loader.LoadCompose(context.Background())
 	if err != nil {
 		t.Fatalf("LoadCompose() failed: %v", err)
@@ -138,15 +137,36 @@ func TestComposeOnlyOneFile(t *testing.T) {
 	})
 	os.Chdir("../../../tests/toomany")
 
-	loader := Loader{}
-	_, err := loader.LoadCompose(context.Background())
-	if err == nil {
-		t.Fatalf("LoadCompose() failed: expected error, got nil")
+	loader := NewLoaderWithPath("")
+	project, err := loader.LoadCompose(context.Background())
+	if err != nil {
+		t.Errorf("LoadCompose() failed: %v", err)
+	}
+
+	if len(project.ComposeFiles) != 1 {
+		t.Errorf("LoadCompose() failed: expected only one config file, got %d", len(project.ComposeFiles))
+	}
+}
+
+func TestComposeMultipleFiles(t *testing.T) {
+	cwd, _ := os.Getwd()
+	t.Cleanup(func() {
+		os.Chdir(cwd)
+	})
+	os.Chdir("../../../tests/multiple")
+
+	composeFiles := []string{"compose1.yaml", "compose2.yaml"}
+	loader := NewLoaderWithOptions(LoaderOptions{ConfigPaths: composeFiles})
+	project, err := loader.LoadCompose(context.Background())
+	if err != nil {
+		t.Fatalf("LoadCompose() failed: %v", err)
+	}
+
+	if len(project.ComposeFiles) != 2 {
+		t.Errorf("LoadCompose() failed: expected 2 compose files, got %d", len(project.ComposeFiles))
 	}
 
-	const expected = `multiple Compose files found: ["./compose.yaml" "./docker-compose.yml"]; use -f to specify which one to use`
-	newCwd, _ := os.Getwd() // make the error message independent of the current working directory
-	if got := strings.ReplaceAll(err.Error(), newCwd, "."); got != expected {
-		t.Errorf("LoadCompose() failed: expected error %q, got: %s", expected, got)
+	if len(project.Services) != 2 {
+		t.Errorf("LoadCompose() failed: expected 2 services, got %d", len(project.Services))
 	}
 }

src/pkg/cli/compose/convert_test.go

@@ -149,7 +149,7 @@ func TestConvertPort(t *testing.T) {
 
 func TestConvert(t *testing.T) {
 	testRunCompose(t, func(t *testing.T, path string) {
-		loader := Loader{path}
+		loader := NewLoaderWithPath(path)
 		proj, err := loader.LoadCompose(context.Background())
 		if err != nil {
 			t.Fatal(err)

src/pkg/cli/compose/loader.go

@@ -2,35 +2,47 @@ package compose
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
-	"path/filepath"
 
 	"github.com/DefangLabs/defang/src/pkg/logs"
 	"github.com/DefangLabs/defang/src/pkg/term"
 	"github.com/DefangLabs/defang/src/pkg/types"
 	"github.com/compose-spec/compose-go/v2/cli"
+	"github.com/compose-spec/compose-go/v2/errdefs"
 	compose "github.com/compose-spec/compose-go/v2/types"
 	"github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v3"
 )
 
+type LoaderOptions struct {
+	ConfigPaths []string
+	WorkingDir  string
+}
+
 type Loader struct {
-	ComposeFilePath string
+	options LoaderOptions
 }
 
-func (c Loader) LoadCompose(ctx context.Context) (*compose.Project, error) {
-	composeFilePath, err := getComposeFilePath(c.ComposeFilePath)
-	if err != nil {
-		return nil, err
+func NewLoaderWithOptions(options LoaderOptions) Loader {
+	return Loader{options: options}
+}
+
+func NewLoaderWithPath(path string) Loader {
+	configPaths := []string{}
+	if path != "" {
+		configPaths = append(configPaths, path)
 	}
-	term.Debug("Loading compose file", composeFilePath)
+	return NewLoaderWithOptions(LoaderOptions{ConfigPaths: configPaths})
+}
 
+func (c Loader) LoadCompose(ctx context.Context) (*compose.Project, error) {
 	// Set logrus send logs via the term package
 	termLogger := logs.TermLogFormatter{Term: term.DefaultTerm}
 	logrus.SetFormatter(termLogger)
 
-	projOpts, err := getDefaultProjectOptions(composeFilePath)
+	projOpts, err := c.projectOptions()
 	if err != nil {
 		return nil, err
 	}
@@ -42,23 +54,27 @@ func (c Loader) LoadCompose(ctx context.Context) (*compose.Project, error) {
 
 	project, err := projOpts.LoadProject(ctx)
 	if err != nil {
+		if errors.Is(err, errdefs.ErrNotFound) {
+			return nil, types.ErrComposeFileNotFound
+		}
+
 		return nil, err
 	}
 
 	if term.DoDebug() {
 		b, _ := yaml.Marshal(project)
 		fmt.Println(string(b))
 	}
+
 	return project, nil
 }
 
-func getDefaultProjectOptions(composeFilePath string, extraOpts ...cli.ProjectOptionsFn) (*cli.ProjectOptions, error) {
-	workingDir := filepath.Dir(composeFilePath)
-
+func (c *Loader) projectOptions() (*cli.ProjectOptions, error) {
+	options := c.options
 	// Based on how docker compose setup its own project options
 	// https://github.com/docker/compose/blob/1a14fcb1e6645dd92f5a4f2da00071bd59c2e887/cmd/compose/compose.go#L326-L346
-	opts := []cli.ProjectOptionsFn{
-		cli.WithWorkingDirectory(workingDir),
+	optFns := []cli.ProjectOptionsFn{
+		cli.WithWorkingDirectory(options.WorkingDir),
 		// First apply os.Environment, always win
 		// -- DISABLED -- cli.WithOsEnv,
 		// Load PWD/.env if present and no explicit --env-file has been set
@@ -68,7 +84,7 @@ func getDefaultProjectOptions(composeFilePath string, extraOpts ...cli.ProjectOp
 		// get compose file path set by COMPOSE_FILE
 		cli.WithConfigFileEnv,
 		// if none was selected, get default compose.yaml file from current dir or parent folder
-		// cli.WithDefaultConfigPath, NO: this ends up picking the "first" when more than one file is found
+		cli.WithDefaultConfigPath,
 		// cli.WithName(o.ProjectName)
 
 		// Calling the 2 functions below the 2nd time as the loaded env in first call modifies the behavior of the 2nd call
@@ -81,55 +97,6 @@ func getDefaultProjectOptions(composeFilePath string, extraOpts ...cli.ProjectOp
 		cli.WithDiscardEnvFile,
 		cli.WithConsistency(false), // TODO: check fails if secrets are used but top-level 'secrets:' is missing
 	}
-	opts = append(opts, extraOpts...)
-	projOpts, err := cli.NewProjectOptions([]string{composeFilePath}, opts...)
-	if err != nil {
-		return nil, err
-	}
 
-	return projOpts, nil
-}
-
-func getComposeFilePath(userSpecifiedComposeFile string) (string, error) {
-	// The Compose file is compose.yaml (preferred) or compose.yml that is placed in the current directory or higher.
-	// Compose also supports docker-compose.yaml and docker-compose.yml for backwards compatibility.
-	// Users can override the file by specifying file name
-	const DEFAULT_COMPOSE_FILE_PATTERN = "*compose.y*ml"
-
-	path, err := os.Getwd()
-	if err != nil {
-		return path, err
-	}
-
-	searchPattern := DEFAULT_COMPOSE_FILE_PATTERN
-	if len(userSpecifiedComposeFile) > 0 {
-		path = ""
-		searchPattern = userSpecifiedComposeFile
-	}
-
-	// iterate through this loop at least once to find the compose file.
-	// if the user did not specify a specific file (i.e. userSpecifiedComposeFile == "")
-	// then walk the tree up to the root directory looking for a compose file.
-	term.Debug("Looking for compose file - searching for", searchPattern)
-	for {
-		if files, _ := filepath.Glob(filepath.Join(path, searchPattern)); len(files) > 1 {
-			return "", fmt.Errorf("multiple Compose files found: %q; use -f to specify which one to use", files)
-		} else if len(files) == 1 {
-			// found compose file, we're done
-			return files[0], nil
-		}
-
-		if len(userSpecifiedComposeFile) > 0 {
-			return "", fmt.Errorf("no Compose file found at %q: %w", userSpecifiedComposeFile, os.ErrNotExist)
-		}
-
-		// compose file not found, try parent directory
-		nextPath := filepath.Dir(path)
-		if nextPath == path {
-			// previous search was of root, we're done
-			return "", types.ErrComposeFileNotFound
-		}
-
-		path = nextPath
-	}
+	return cli.NewProjectOptions(options.ConfigPaths, optFns...)
 }

src/pkg/cli/compose/loader_test.go

@@ -16,11 +16,12 @@ import (
 
 func TestLoader(t *testing.T) {
 	testRunCompose(t, func(t *testing.T, path string) {
-		loader := Loader{path}
+		loader := NewLoaderWithPath(path)
 		proj, err := loader.LoadCompose(context.Background())
 		if err != nil {
 			t.Fatal(err)
 		}
+
 		yaml, err := proj.MarshalYAML()
 		if err != nil {
 			t.Fatal(err)

src/pkg/cli/compose/validation_test.go

@@ -22,7 +22,8 @@ func TestValidationAndConvert(t *testing.T) {
 		logs := new(bytes.Buffer)
 		term.DefaultTerm = term.NewTerm(logs, logs)
 
-		loader := Loader{path}
+		options := LoaderOptions{ConfigPaths: []string{path}}
+		loader := Loader{options: options}
 		proj, err := loader.LoadCompose(context.Background())
 		if err != nil {
 			t.Fatal(err)