Merge pull request #435 from ndeloof/remote_resource

Signed-off-by: Nicolas De Loof <[email protected]>

Author: ndeloof

cli/options.go

@@ -17,6 +17,7 @@
 package cli
 
 import (
+	"context"
 	"io"
 	"os"
 	"path/filepath"
@@ -35,6 +36,8 @@ import (
 
 // ProjectOptions provides common configuration for loading a project.
 type ProjectOptions struct {
+	ctx context.Context
+
 	// Name is a valid Compose project name to be used or empty.
 	//
 	// If empty, the project loader will automatically infer a reasonable
@@ -301,6 +304,24 @@ func WithResolvedPaths(resolve bool) ProjectOptionsFn {
 	}
 }
 
+// WithContext sets the context used to load model and resources
+func WithContext(ctx context.Context) ProjectOptionsFn {
+	return func(o *ProjectOptions) error {
+		o.ctx = ctx
+		return nil
+	}
+}
+
+// WithResourceLoader register support for ResourceLoader to manage remote resources
+func WithResourceLoader(r loader.ResourceLoader) ProjectOptionsFn {
+	return func(o *ProjectOptions) error {
+		o.loadOptions = append(o.loadOptions, func(options *loader.Options) {
+			options.ResourceLoaders = append(options.ResourceLoaders, r)
+		})
+		return nil
+	}
+}
+
 // DefaultFileNames defines the Compose file names for auto-discovery (in order of preference)
 var DefaultFileNames = []string{"compose.yaml", "compose.yml", "docker-compose.yml", "docker-compose.yaml"}
 
@@ -367,7 +388,12 @@ func ProjectFromOptions(options *ProjectOptions) (*types.Project, error) {
 		withNamePrecedenceLoad(absWorkingDir, options),
 		withConvertWindowsPaths(options))
 
-	project, err := loader.Load(types.ConfigDetails{
+	ctx := options.ctx
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
+	project, err := loader.LoadWithContext(ctx, types.ConfigDetails{
 		ConfigFiles: configs,
 		WorkingDir:  workingDir,
 		Environment: options.Environment,

loader/include.go

@@ -17,10 +17,12 @@
 package loader
 
 import (
+	"context"
 	"fmt"
 	"path/filepath"
 
 	"github.com/compose-spec/compose-go/dotenv"
+	interp "github.com/compose-spec/compose-go/interpolation"
 	"github.com/compose-spec/compose-go/types"
 	"github.com/pkg/errors"
 )
@@ -43,12 +45,20 @@ var transformIncludeConfig TransformerFunc = func(data interface{}) (interface{}
 	}
 }
 
-func loadInclude(configDetails types.ConfigDetails, model *types.Config, options *Options, loaded []string) (*types.Config, error) {
+func loadInclude(ctx context.Context, configDetails types.ConfigDetails, model *types.Config, options *Options, loaded []string) (*types.Config, error) {
 	for _, r := range model.Include {
 		for i, p := range r.Path {
-			if !filepath.IsAbs(p) {
-				r.Path[i] = filepath.Join(configDetails.WorkingDir, p)
+			for _, loader := range options.ResourceLoaders {
+				if loader.Accept(p) {
+					path, err := loader.Load(ctx, p)
+					if err != nil {
+						return nil, err
+					}
+					p = path
+					break
+				}
 			}
+			r.Path[i] = absPath(configDetails.WorkingDir, p)
 		}
 		if r.ProjectDirectory == "" {
 			r.ProjectDirectory = filepath.Dir(r.Path[0])
@@ -65,11 +75,17 @@ func loadInclude(configDetails types.ConfigDetails, model *types.Config, options
 			return nil, err
 		}
 
-		imported, err := load(types.ConfigDetails{
+		config := types.ConfigDetails{
 			WorkingDir:  r.ProjectDirectory,
 			ConfigFiles: types.ToConfigFiles(r.Path),
 			Environment: env,
-		}, loadOptions, loaded)
+		}
+		loadOptions.Interpolate = &interp.Options{
+			Substitute:      options.Interpolate.Substitute,
+			LookupValue:     config.LookupEnv,
+			TypeCastMapping: options.Interpolate.TypeCastMapping,
+		}
+		imported, err := load(ctx, config, loadOptions, loaded)
 		if err != nil {
 			return nil, err
 		}

loader/loader.go

@@ -17,6 +17,7 @@
 package loader
 
 import (
+	"context"
 	"fmt"
 	"os"
 	paths "path"
@@ -68,6 +69,16 @@ type Options struct {
 	projectNameImperativelySet bool
 	// Profiles set profiles to enable
 	Profiles []string
+	// ResourceLoaders manages support for remote resources
+	ResourceLoaders []ResourceLoader
+}
+
+// ResourceLoader is a plugable remote resource resolver
+type ResourceLoader interface {
+	// Accept returns `true` is the resource reference matches ResourceLoader supported protocol(s)
+	Accept(path string) bool
+	// Load returns the path to a local copy of remote resource identified by `path`.
+	Load(ctx context.Context, path string) (string, error)
 }
 
 func (o *Options) clone() *Options {
@@ -85,6 +96,7 @@ func (o *Options) clone() *Options {
 		projectName:                o.projectName,
 		projectNameImperativelySet: o.projectNameImperativelySet,
 		Profiles:                   o.Profiles,
+		ResourceLoaders:            o.ResourceLoaders,
 	}
 }
 
@@ -193,8 +205,14 @@ func parseYAML(source []byte) (map[string]interface{}, PostProcessor, error) {
 	return converted.(map[string]interface{}), &processor, nil
 }
 
-// Load reads a ConfigDetails and returns a fully loaded configuration
+// Load reads a ConfigDetails and returns a fully loaded configuration.
+// Deprecated: use LoadWithContext.
 func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types.Project, error) {
+	return LoadWithContext(context.Background(), configDetails, options...)
+}
+
+// LoadWithContext reads a ConfigDetails and returns a fully loaded configuration
+func LoadWithContext(ctx context.Context, configDetails types.ConfigDetails, options ...func(*Options)) (*types.Project, error) {
 	if len(configDetails.ConfigFiles) < 1 {
 		return nil, errors.Errorf("No files specified")
 	}
@@ -217,10 +235,10 @@ func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types.
 		return nil, err
 	}
 	opts.projectName = projectName
-	return load(configDetails, opts, nil)
+	return load(ctx, configDetails, opts, nil)
 }
 
-func load(configDetails types.ConfigDetails, opts *Options, loaded []string) (*types.Project, error) {
+func load(ctx context.Context, configDetails types.ConfigDetails, opts *Options, loaded []string) (*types.Project, error) {
 	var model *types.Config
 
 	mainFile := configDetails.ConfigFiles[0].Filename
@@ -261,13 +279,13 @@ func load(configDetails types.ConfigDetails, opts *Options, loaded []string) (*t
 
 		configDict = groupXFieldsIntoExtensions(configDict)
 
-		cfg, err := loadSections(file.Filename, configDict, configDetails, opts)
+		cfg, err := loadSections(ctx, file.Filename, configDict, configDetails, opts)
 		if err != nil {
 			return nil, err
 		}
 
 		if !opts.SkipInclude {
-			cfg, err = loadInclude(configDetails, cfg, opts, loaded)
+			cfg, err = loadInclude(ctx, configDetails, cfg, opts, loaded)
 			if err != nil {
 				return nil, err
 			}
@@ -453,7 +471,7 @@ func groupXFieldsIntoExtensions(dict map[string]interface{}) map[string]interfac
 	return dict
 }
 
-func loadSections(filename string, config map[string]interface{}, configDetails types.ConfigDetails, opts *Options) (*types.Config, error) {
+func loadSections(ctx context.Context, filename string, config map[string]interface{}, configDetails types.ConfigDetails, opts *Options) (*types.Config, error) {
 	var err error
 	cfg := types.Config{
 		Filename: filename,
@@ -466,7 +484,7 @@ func loadSections(filename string, config map[string]interface{}, configDetails
 		}
 	}
 	cfg.Name = name
-	cfg.Services, err = LoadServices(filename, getSection(config, "services"), configDetails.WorkingDir, configDetails.LookupEnv, opts)
+	cfg.Services, err = LoadServices(ctx, filename, getSection(config, "services"), configDetails.WorkingDir, configDetails.LookupEnv, opts)
 	if err != nil {
 		return nil, err
 	}
@@ -659,7 +677,7 @@ func formatInvalidKeyError(keyPrefix string, key interface{}) error {
 
 // LoadServices produces a ServiceConfig map from a compose file Dict
 // the servicesDict is not validated if directly used. Use Load() to enable validation
-func LoadServices(filename string, servicesDict map[string]interface{}, workingDir string, lookupEnv template.Mapping, opts *Options) ([]types.ServiceConfig, error) {
+func LoadServices(ctx context.Context, filename string, servicesDict map[string]interface{}, workingDir string, lookupEnv template.Mapping, opts *Options) ([]types.ServiceConfig, error) {
 	var services []types.ServiceConfig
 
 	x, ok := servicesDict[extensions]
@@ -672,7 +690,7 @@ func LoadServices(filename string, servicesDict map[string]interface{}, workingD
 	}
 
 	for name := range servicesDict {
-		serviceConfig, err := loadServiceWithExtends(filename, name, servicesDict, workingDir, lookupEnv, opts, &cycleTracker{})
+		serviceConfig, err := loadServiceWithExtends(ctx, filename, name, servicesDict, workingDir, lookupEnv, opts, &cycleTracker{})
 		if err != nil {
 			return nil, err
 		}
@@ -683,7 +701,7 @@ func LoadServices(filename string, servicesDict map[string]interface{}, workingD
 	return services, nil
 }
 
-func loadServiceWithExtends(filename, name string, servicesDict map[string]interface{}, workingDir string, lookupEnv template.Mapping, opts *Options, ct *cycleTracker) (*types.ServiceConfig, error) {
+func loadServiceWithExtends(ctx context.Context, filename, name string, servicesDict map[string]interface{}, workingDir string, lookupEnv template.Mapping, opts *Options, ct *cycleTracker) (*types.ServiceConfig, error) {
 	if err := ct.Add(filename, name); err != nil {
 		return nil, err
 	}
@@ -707,11 +725,21 @@ func loadServiceWithExtends(filename, name string, servicesDict map[string]inter
 		var baseService *types.ServiceConfig
 		file := serviceConfig.Extends.File
 		if file == "" {
-			baseService, err = loadServiceWithExtends(filename, baseServiceName, servicesDict, workingDir, lookupEnv, opts, ct)
+			baseService, err = loadServiceWithExtends(ctx, filename, baseServiceName, servicesDict, workingDir, lookupEnv, opts, ct)
 			if err != nil {
 				return nil, err
 			}
 		} else {
+			for _, loader := range opts.ResourceLoaders {
+				if loader.Accept(file) {
+					path, err := loader.Load(ctx, file)
+					if err != nil {
+						return nil, err
+					}
+					file = path
+					break
+				}
+			}
 			// Resolve the path to the imported file, and load it.
 			baseFilePath := absPath(workingDir, file)
 
@@ -726,7 +754,7 @@ func loadServiceWithExtends(filename, name string, servicesDict map[string]inter
 			}
 
 			baseFileServices := getSection(baseFile, "services")
-			baseService, err = loadServiceWithExtends(baseFilePath, baseServiceName, baseFileServices, filepath.Dir(baseFilePath), lookupEnv, opts, ct)
+			baseService, err = loadServiceWithExtends(ctx, baseFilePath, baseServiceName, baseFileServices, filepath.Dir(baseFilePath), lookupEnv, opts, ct)
 			if err != nil {
 				return nil, err
 			}