load: include details about included files on *Project (#444)

Add a map to `Project` that has Compose YAML filename as key and
any `IncludeConfig`s loaded from it.

This is populated as the project is recursively loaded.

For example:
```
  proj/
    compose.yaml
    a/
      compose.yaml
    b/
      compose.yaml
```

If `project/compose.yaml` has:
```
include: ['./a/compose.yaml']
```

And `project/a/compose.yaml` has:
```
include: ['../b/compose.yaml']
```

The final result after load is conceptually:
```
{
  "proj/compose.yaml": ["proj/a/compose.yaml"],
  "proj/a/compose.yaml": ["proj/b/compose.yaml"],
}
```
(Note: in reality, it's a list of `IncludeConfig`, which has multiple
fields. Example is simplified.)

Relative path resolution is based on overall loader configuration,
but note that disabling it does not work properly for `include`
currently due to other issues.

This makes it possible for the caller to understand a bit more
about the loaded project resources. We're really overdue for a bit
of an overhaul/refactor of the loader - at that point, I think it
might be better to have a `Loader` object type that can track stuff
like this on the instance because it's weirdly both part of the
project and NOT part of the project at the moment (similar to
`Profiles`, project name, etc).

Signed-off-by: Milas Bowman <[email protected]>

Author: milas

loader/include.go

@@ -45,14 +45,17 @@ var transformIncludeConfig TransformerFunc = func(data interface{}) (interface{}
 	}
 }
 
-func loadInclude(ctx context.Context, configDetails types.ConfigDetails, model *types.Config, options *Options, loaded []string) (*types.Config, error) {
+func loadInclude(ctx context.Context, filename string, configDetails types.ConfigDetails, model *types.Config, options *Options, loaded []string) (*types.Config, map[string][]types.IncludeConfig, error) {
+	included := make(map[string][]types.IncludeConfig)
 	for _, r := range model.Include {
+		included[filename] = append(included[filename], r)
+
 		for i, p := range r.Path {
 			for _, loader := range options.ResourceLoaders {
 				if loader.Accept(p) {
 					path, err := loader.Load(ctx, p)
 					if err != nil {
-						return nil, err
+						return nil, nil, err
 					}
 					p = path
 					break
@@ -72,7 +75,7 @@ func loadInclude(ctx context.Context, configDetails types.ConfigDetails, model *
 
 		env, err := dotenv.GetEnvFromFile(configDetails.Environment, r.ProjectDirectory, r.EnvFile)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 
 		config := types.ConfigDetails{
@@ -87,16 +90,19 @@ func loadInclude(ctx context.Context, configDetails types.ConfigDetails, model *
 		}
 		imported, err := load(ctx, config, loadOptions, loaded)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
+		}
+		for k, v := range imported.IncludeReferences {
+			included[k] = append(included[k], v...)
 		}
 
 		err = importResources(model, imported, r.Path)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 	}
 	model.Include = nil
-	return model, nil
+	return model, included, nil
 }
 
 // importResources import into model all resources defined by imported, and report error on conflict

loader/loader.go

@@ -250,6 +250,7 @@ func load(ctx context.Context, configDetails types.ConfigDetails, opts *Options,
 	}
 	loaded = append(loaded, mainFile)
 
+	includeRefs := make(map[string][]types.IncludeConfig)
 	for i, file := range configDetails.ConfigFiles {
 		var postProcessor PostProcessor
 		configDict := file.Config
@@ -285,10 +286,14 @@ func load(ctx context.Context, configDetails types.ConfigDetails, opts *Options,
 		}
 
 		if !opts.SkipInclude {
-			cfg, err = loadInclude(ctx, configDetails, cfg, opts, loaded)
+			var included map[string][]types.IncludeConfig
+			cfg, included, err = loadInclude(ctx, file.Filename, configDetails, cfg, opts, loaded)
 			if err != nil {
 				return nil, err
 			}
+			for k, v := range included {
+				includeRefs[k] = append(includeRefs[k], v...)
+			}
 		}
 
 		if i == 0 {
@@ -321,6 +326,10 @@ func load(ctx context.Context, configDetails types.ConfigDetails, opts *Options,
 		Extensions:  model.Extensions,
 	}
 
+	if len(includeRefs) != 0 {
+		project.IncludeReferences = includeRefs
+	}
+
 	if !opts.SkipNormalization {
 		err := Normalize(project)
 		if err != nil {

loader/loader_test.go

@@ -2545,6 +2545,15 @@ services:
 			},
 		},
 	})
+	assert.DeepEqual(t, p.IncludeReferences, map[string][]types.IncludeConfig{
+		filepath.Join(workingDir, "filename0.yml"): {
+			{
+				Path:             []string{filepath.Join(workingDir, "testdata", "subdir", "compose-test-extends-imported.yaml")},
+				ProjectDirectory: workingDir,
+				EnvFile:          []string{filepath.Join(workingDir, "testdata", "subdir", "extra.env")},
+			},
+		},
+	})
 	assert.NilError(t, err)
 }
 

loader/paths.go

@@ -64,6 +64,25 @@ func ResolveRelativePaths(project *types.Project) error {
 			project.Volumes[name] = config
 		}
 	}
+
+	// don't coerce a nil map to an empty map
+	if project.IncludeReferences != nil {
+		absIncludes := make(map[string][]types.IncludeConfig, len(project.IncludeReferences))
+		for filename, config := range project.IncludeReferences {
+			filename = absPath(project.WorkingDir, filename)
+			absConfigs := make([]types.IncludeConfig, len(config))
+			for i, c := range config {
+				absConfigs[i] = types.IncludeConfig{
+					Path:             resolvePaths(project.WorkingDir, c.Path),
+					ProjectDirectory: absPath(project.WorkingDir, c.ProjectDirectory),
+					EnvFile:          resolvePaths(project.WorkingDir, c.EnvFile),
+				}
+			}
+			absIncludes[filename] = absConfigs
+		}
+		project.IncludeReferences = absIncludes
+	}
+
 	return nil
 }
 
@@ -133,3 +152,14 @@ func isRemoteContext(maybeURL string) bool {
 	}
 	return false
 }
+
+func resolvePaths(basePath string, in types.StringList) types.StringList {
+	if in == nil {
+		return nil
+	}
+	ret := make(types.StringList, len(in))
+	for i := range in {
+		ret[i] = absPath(basePath, in[i])
+	}
+	return ret
+}

types/project.go

@@ -36,16 +36,22 @@ import (
 
 // Project is the result of loading a set of compose files
 type Project struct {
-	Name         string     `yaml:"name,omitempty" json:"name,omitempty"`
-	WorkingDir   string     `yaml:"-" json:"-"`
-	Services     Services   `yaml:"services" json:"services"`
-	Networks     Networks   `yaml:"networks,omitempty" json:"networks,omitempty"`
-	Volumes      Volumes    `yaml:"volumes,omitempty" json:"volumes,omitempty"`
-	Secrets      Secrets    `yaml:"secrets,omitempty" json:"secrets,omitempty"`
-	Configs      Configs    `yaml:"configs,omitempty" json:"configs,omitempty"`
-	Extensions   Extensions `yaml:"#extensions,inline" json:"-"` // https://github.com/golang/go/issues/6213
-	ComposeFiles []string   `yaml:"-" json:"-"`
-	Environment  Mapping    `yaml:"-" json:"-"`
+	Name       string     `yaml:"name,omitempty" json:"name,omitempty"`
+	WorkingDir string     `yaml:"-" json:"-"`
+	Services   Services   `yaml:"services" json:"services"`
+	Networks   Networks   `yaml:"networks,omitempty" json:"networks,omitempty"`
+	Volumes    Volumes    `yaml:"volumes,omitempty" json:"volumes,omitempty"`
+	Secrets    Secrets    `yaml:"secrets,omitempty" json:"secrets,omitempty"`
+	Configs    Configs    `yaml:"configs,omitempty" json:"configs,omitempty"`
+	Extensions Extensions `yaml:"#extensions,inline" json:"-"` // https://github.com/golang/go/issues/6213
+
+	// IncludeReferences is keyed by Compose YAML filename and contains config for
+	// other Compose YAML files it directly triggered a load of via `include`.
+	//
+	// Note: this is
+	IncludeReferences map[string][]IncludeConfig `yaml:"-" json:"-"`
+	ComposeFiles      []string                   `yaml:"-" json:"-"`
+	Environment       Mapping                    `yaml:"-" json:"-"`
 
 	// DisabledServices track services which have been disable as profile is not active
 	DisabledServices Services `yaml:"-" json:"-"`