Skip to content

Commit ad9be94

Browse files
ndeloofndeloof
committed
introduce ResourceResolver to accept remote resources
Signed-off-by: Nicolas De Loof <[email protected]>
1 parent 08d8d55 commit ad9be94

File tree

10 files changed

+223
-17
lines changed

10 files changed

+223
-17
lines changed

‎cli/options.go‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package cli
1818

1919
import (
20+
"context"
2021
"io"
2122
"os"
2223
"path/filepath"
@@ -35,6 +36,8 @@ import (
3536

3637
// ProjectOptions provides common configuration for loading a project.
3738
type ProjectOptions struct {
39+
ctx context.Context
40+
3841
// Name is a valid Compose project name to be used or empty.
3942
//
4043
// If empty, the project loader will automatically infer a reasonable
@@ -301,6 +304,24 @@ func WithResolvedPaths(resolve bool) ProjectOptionsFn {
301304
}
302305
}
303306

307+
// WithContext sets the context used to load model and resources
308+
func WithContext(ctx context.Context) ProjectOptionsFn {
309+
return func(o *ProjectOptions) error {
310+
o.ctx = ctx
311+
return nil
312+
}
313+
}
314+
315+
// WithResourceLoader register support for ResourceLoader to manage remote resources
316+
func WithResourceLoader(r loader.ResourceLoader) ProjectOptionsFn {
317+
return func(o *ProjectOptions) error {
318+
o.loadOptions = append(o.loadOptions, func(options *loader.Options) {
319+
options.ResourceLoaders = append(options.ResourceLoaders, r)
320+
})
321+
return nil
322+
}
323+
}
324+
304325
// DefaultFileNames defines the Compose file names for auto-discovery (in order of preference)
305326
var DefaultFileNames = []string{"compose.yaml", "compose.yml", "docker-compose.yml", "docker-compose.yaml"}
306327

@@ -367,7 +388,12 @@ func ProjectFromOptions(options *ProjectOptions) (*types.Project, error) {
367388
withNamePrecedenceLoad(absWorkingDir, options),
368389
withConvertWindowsPaths(options))
369390

370-
project, err := loader.Load(types.ConfigDetails{
391+
ctx := options.ctx
392+
if ctx == nil {
393+
ctx = context.Background()
394+
}
395+
396+
project, err := loader.LoadWithContext(ctx, types.ConfigDetails{
371397
ConfigFiles: configs,
372398
WorkingDir: workingDir,
373399
Environment: options.Environment,

‎loader/include.go‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package loader
1818

1919
import (
20+
"context"
2021
"fmt"
2122
"path/filepath"
2223

@@ -43,12 +44,20 @@ var transformIncludeConfig TransformerFunc = func(data interface{}) (interface{}
4344
}
4445
}
4546

46-
func loadInclude(configDetails types.ConfigDetails, model *types.Config, options *Options, loaded []string) (*types.Config, error) {
47+
func loadInclude(ctx context.Context, configDetails types.ConfigDetails, model *types.Config, options *Options, loaded []string) (*types.Config, error) {
4748
for _, r := range model.Include {
4849
for i, p := range r.Path {
49-
if !filepath.IsAbs(p) {
50-
r.Path[i] = filepath.Join(configDetails.WorkingDir, p)
50+
for _, loader := range options.ResourceLoaders {
51+
if loader.Accept(p) {
52+
path, err := loader.Load(ctx, p)
53+
if err != nil {
54+
return nil, err
55+
}
56+
p = path
57+
break
58+
}
5159
}
60+
r.Path[i] = absPath(configDetails.WorkingDir, p)
5261
}
5362
if r.ProjectDirectory == "" {
5463
r.ProjectDirectory = filepath.Dir(r.Path[0])
@@ -65,7 +74,7 @@ func loadInclude(configDetails types.ConfigDetails, model *types.Config, options
6574
return nil, err
6675
}
6776

68-
imported, err := load(types.ConfigDetails{
77+
imported, err := load(ctx, types.ConfigDetails{
6978
WorkingDir: r.ProjectDirectory,
7079
ConfigFiles: types.ToConfigFiles(r.Path),
7180
Environment: env,

‎loader/loader.go‎

Lines changed: 40 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package loader
1818

1919
import (
20+
"context"
2021
"fmt"
2122
"os"
2223
paths "path"
@@ -68,6 +69,16 @@ type Options struct {
6869
projectNameImperativelySet bool
6970
// Profiles set profiles to enable
7071
Profiles []string
72+
// ResourceLoaders manages support for remote resources
73+
ResourceLoaders []ResourceLoader
74+
}
75+
76+
// ResourceLoader is a plugable remote resource resolver
77+
type ResourceLoader interface {
78+
// Accept returns `true` is the resource reference matches ResourceLoader supported protocol(s)
79+
Accept(path string) bool
80+
// Load returns the path to a local copy of remote resource identified by `path`.
81+
Load(ctx context.Context, path string) (string, error)
7182
}
7283

7384
func (o *Options) clone() *Options {
@@ -85,6 +96,7 @@ func (o *Options) clone() *Options {
8596
projectName: o.projectName,
8697
projectNameImperativelySet: o.projectNameImperativelySet,
8798
Profiles: o.Profiles,
99+
ResourceLoaders: o.ResourceLoaders,
88100
}
89101
}
90102

@@ -193,8 +205,14 @@ func parseYAML(source []byte) (map[string]interface{}, PostProcessor, error) {
193205
return converted.(map[string]interface{}), &processor, nil
194206
}
195207

196-
// Load reads a ConfigDetails and returns a fully loaded configuration
208+
// Load reads a ConfigDetails and returns a fully loaded configuration.
209+
// Deprecated: use LoadWithContext.
197210
func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types.Project, error) {
211+
return LoadWithContext(context.Background(), configDetails, options...)
212+
}
213+
214+
// LoadWithContext reads a ConfigDetails and returns a fully loaded configuration
215+
func LoadWithContext(ctx context.Context, configDetails types.ConfigDetails, options ...func(*Options)) (*types.Project, error) {
198216
if len(configDetails.ConfigFiles) < 1 {
199217
return nil, errors.Errorf("No files specified")
200218
}
@@ -217,10 +235,10 @@ func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types.
217235
return nil, err
218236
}
219237
opts.projectName = projectName
220-
return load(configDetails, opts, nil)
238+
return load(ctx, configDetails, opts, nil)
221239
}
222240

223-
func load(configDetails types.ConfigDetails, opts *Options, loaded []string) (*types.Project, error) {
241+
func load(ctx context.Context, configDetails types.ConfigDetails, opts *Options, loaded []string) (*types.Project, error) {
224242
var model *types.Config
225243

226244
mainFile := configDetails.ConfigFiles[0].Filename
@@ -261,13 +279,13 @@ func load(configDetails types.ConfigDetails, opts *Options, loaded []string) (*t
261279

262280
configDict = groupXFieldsIntoExtensions(configDict)
263281

264-
cfg, err := loadSections(file.Filename, configDict, configDetails, opts)
282+
cfg, err := loadSections(ctx, file.Filename, configDict, configDetails, opts)
265283
if err != nil {
266284
return nil, err
267285
}
268286

269287
if !opts.SkipInclude {
270-
cfg, err = loadInclude(configDetails, cfg, opts, loaded)
288+
cfg, err = loadInclude(ctx, configDetails, cfg, opts, loaded)
271289
if err != nil {
272290
return nil, err
273291
}
@@ -453,7 +471,7 @@ func groupXFieldsIntoExtensions(dict map[string]interface{}) map[string]interfac
453471
return dict
454472
}
455473

456-
func loadSections(filename string, config map[string]interface{}, configDetails types.ConfigDetails, opts *Options) (*types.Config, error) {
474+
func loadSections(ctx context.Context, filename string, config map[string]interface{}, configDetails types.ConfigDetails, opts *Options) (*types.Config, error) {
457475
var err error
458476
cfg := types.Config{
459477
Filename: filename,
@@ -466,7 +484,7 @@ func loadSections(filename string, config map[string]interface{}, configDetails
466484
}
467485
}
468486
cfg.Name = name
469-
cfg.Services, err = LoadServices(filename, getSection(config, "services"), configDetails.WorkingDir, configDetails.LookupEnv, opts)
487+
cfg.Services, err = LoadServices(ctx, filename, getSection(config, "services"), configDetails.WorkingDir, configDetails.LookupEnv, opts)
470488
if err != nil {
471489
return nil, err
472490
}
@@ -659,7 +677,7 @@ func formatInvalidKeyError(keyPrefix string, key interface{}) error {
659677

660678
// LoadServices produces a ServiceConfig map from a compose file Dict
661679
// the servicesDict is not validated if directly used. Use Load() to enable validation
662-
func LoadServices(filename string, servicesDict map[string]interface{}, workingDir string, lookupEnv template.Mapping, opts *Options) ([]types.ServiceConfig, error) {
680+
func LoadServices(ctx context.Context, filename string, servicesDict map[string]interface{}, workingDir string, lookupEnv template.Mapping, opts *Options) ([]types.ServiceConfig, error) {
663681
var services []types.ServiceConfig
664682

665683
x, ok := servicesDict[extensions]
@@ -672,7 +690,7 @@ func LoadServices(filename string, servicesDict map[string]interface{}, workingD
672690
}
673691

674692
for name := range servicesDict {
675-
serviceConfig, err := loadServiceWithExtends(filename, name, servicesDict, workingDir, lookupEnv, opts, &cycleTracker{})
693+
serviceConfig, err := loadServiceWithExtends(ctx, filename, name, servicesDict, workingDir, lookupEnv, opts, &cycleTracker{})
676694
if err != nil {
677695
return nil, err
678696
}
@@ -683,7 +701,7 @@ func LoadServices(filename string, servicesDict map[string]interface{}, workingD
683701
return services, nil
684702
}
685703

686-
func loadServiceWithExtends(filename, name string, servicesDict map[string]interface{}, workingDir string, lookupEnv template.Mapping, opts *Options, ct *cycleTracker) (*types.ServiceConfig, error) {
704+
func loadServiceWithExtends(ctx context.Context, filename, name string, servicesDict map[string]interface{}, workingDir string, lookupEnv template.Mapping, opts *Options, ct *cycleTracker) (*types.ServiceConfig, error) {
687705
if err := ct.Add(filename, name); err != nil {
688706
return nil, err
689707
}
@@ -707,11 +725,21 @@ func loadServiceWithExtends(filename, name string, servicesDict map[string]inter
707725
var baseService *types.ServiceConfig
708726
file := serviceConfig.Extends.File
709727
if file == "" {
710-
baseService, err = loadServiceWithExtends(filename, baseServiceName, servicesDict, workingDir, lookupEnv, opts, ct)
728+
baseService, err = loadServiceWithExtends(ctx, filename, baseServiceName, servicesDict, workingDir, lookupEnv, opts, ct)
711729
if err != nil {
712730
return nil, err
713731
}
714732
} else {
733+
for _, loader := range opts.ResourceLoaders {
734+
if loader.Accept(file) {
735+
path, err := loader.Load(ctx, file)
736+
if err != nil {
737+
return nil, err
738+
}
739+
file = path
740+
break
741+
}
742+
}
715743
// Resolve the path to the imported file, and load it.
716744
baseFilePath := absPath(workingDir, file)
717745

@@ -726,7 +754,7 @@ func loadServiceWithExtends(filename, name string, servicesDict map[string]inter
726754
}
727755

728756
baseFileServices := getSection(baseFile, "services")
729-
baseService, err = loadServiceWithExtends(baseFilePath, baseServiceName, baseFileServices, filepath.Dir(baseFilePath), lookupEnv, opts, ct)
757+
baseService, err = loadServiceWithExtends(ctx, baseFilePath, baseServiceName, baseFileServices, filepath.Dir(baseFilePath), lookupEnv, opts, ct)
730758
if err != nil {
731759
return nil, err
732760
}

‎loader/loader_test.go‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package loader
1818

1919
import (
2020
"bytes"
21+
"context"
2122
"fmt"
2223
"os"
2324
"path/filepath"
@@ -2592,3 +2593,119 @@ services:
25922593
},
25932594
})
25942595
}
2596+
2597+
type customLoader struct {
2598+
prefix string
2599+
}
2600+
2601+
func (c customLoader) Accept(s string) bool {
2602+
return strings.HasPrefix(s, c.prefix+":")
2603+
}
2604+
2605+
func (c customLoader) Load(ctx context.Context, s string) (string, error) {
2606+
path := filepath.Join("testdata", c.prefix, s[len(c.prefix)+1:])
2607+
_, err := os.Stat(path)
2608+
if err != nil {
2609+
return "", err
2610+
}
2611+
return filepath.Abs(path)
2612+
}
2613+
2614+
func TestLoadWithRemoteResources(t *testing.T) {
2615+
config := buildConfigDetails(`
2616+
name: test-remote-resources
2617+
services:
2618+
foo:
2619+
extends:
2620+
file: remote:compose.yaml
2621+
service: foo
2622+
2623+
`, nil)
2624+
p, err := LoadWithContext(context.Background(), config, func(options *Options) {
2625+
options.SkipConsistencyCheck = true
2626+
options.SkipNormalization = true
2627+
options.ResolvePaths = true
2628+
options.ResourceLoaders = []ResourceLoader{
2629+
customLoader{prefix: "remote"},
2630+
}
2631+
})
2632+
assert.NilError(t, err)
2633+
assert.DeepEqual(t, p.Services, types.Services{
2634+
{
2635+
Name: "foo",
2636+
Image: "foo",
2637+
Environment: types.MappingWithEquals{"FOO": strPtr("BAR")},
2638+
EnvFile: types.StringList{
2639+
filepath.Join(config.WorkingDir, "testdata", "remote", "env"),
2640+
},
2641+
Scale: 1,
2642+
Volumes: []types.ServiceVolumeConfig{
2643+
{
2644+
Type: types.VolumeTypeBind,
2645+
Source: filepath.Join(config.WorkingDir, "testdata", "remote"),
2646+
Target: "/foo",
2647+
Bind: &types.ServiceVolumeBind{CreateHostPath: true},
2648+
},
2649+
},
2650+
},
2651+
})
2652+
}
2653+
2654+
func TestLoadWithMissingResources(t *testing.T) {
2655+
config := buildConfigDetails(`
2656+
name: test-missing-resources
2657+
services:
2658+
foo:
2659+
extends:
2660+
file: remote:unavailable.yaml
2661+
service: foo
2662+
2663+
`, nil)
2664+
_, err := LoadWithContext(context.Background(), config, func(options *Options) {
2665+
options.SkipConsistencyCheck = true
2666+
options.SkipNormalization = true
2667+
options.ResolvePaths = true
2668+
options.ResourceLoaders = []ResourceLoader{
2669+
customLoader{prefix: "remote"},
2670+
}
2671+
})
2672+
assert.Check(t, os.IsNotExist(err))
2673+
}
2674+
2675+
func TestLoadWithNestedResources(t *testing.T) {
2676+
config := buildConfigDetails(`
2677+
name: test-nested-resources
2678+
include:
2679+
- remote:nested/compose.yaml
2680+
`, nil)
2681+
_, err := LoadWithContext(context.Background(), config, func(options *Options) {
2682+
options.SkipConsistencyCheck = true
2683+
options.SkipNormalization = true
2684+
options.ResolvePaths = true
2685+
options.ResourceLoaders = []ResourceLoader{
2686+
customLoader{prefix: "remote"},
2687+
}
2688+
})
2689+
assert.NilError(t, err)
2690+
}
2691+
2692+
func TestLoadWithResourcesCycle(t *testing.T) {
2693+
config := buildConfigDetails(`
2694+
name: test-resources-cycle
2695+
services:
2696+
foo:
2697+
extends:
2698+
file: remote:cycle/compose.yaml
2699+
service: foo
2700+
2701+
`, nil)
2702+
_, err := LoadWithContext(context.Background(), config, func(options *Options) {
2703+
options.SkipConsistencyCheck = true
2704+
options.SkipNormalization = true
2705+
options.ResolvePaths = true
2706+
options.ResourceLoaders = []ResourceLoader{
2707+
customLoader{prefix: "remote"},
2708+
}
2709+
})
2710+
assert.ErrorContains(t, err, "Circular reference")
2711+
}

‎loader/testdata/remote/compose.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
services:
2+
foo:
3+
image: foo
4+
env_file:
5+
- ./env
6+
volumes:
7+
- .:/foo

‎loader/testdata/remote/cycle/compose-cycle.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
services:
2+
bar:
3+
extends:
4+
file: remote:cycle/compose.yaml
5+
service: foo

‎loader/testdata/remote/cycle/compose.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
services:
2+
foo:
3+
extends:
4+
file: remote:cycle/compose-cycle.yaml
5+
service: bar

0 commit comments

Comments
 (0)