add: refactor custom build & add disabled config options

FabianKramm · FabianKramm · commit 9747ce39b58e · 2021-12-13T13:17:45.000+01:00
diff --git a/examples/custom-builder/custom/build b/examples/custom-builder/custom/build
diff --git a/examples/custom-builder/custom/build.cmd b/examples/custom-builder/custom/build.cmd
diff --git a/examples/custom-builder/devspace.yaml b/examples/custom-builder/devspace.yaml
@@ -9,15 +9,10 @@ images:
     image: ${IMAGE}
     build:
       custom:
-        command: ./custom/build
-        # command: docker
-        # args:
-        # - build
-        # - .
-        # - --tag
-        # appendArgs:
-        # - --file
-        # - ./custom/Dockerfile
+        skipImageArg: true
+        command: |-
+          # Also works on windows as this is executed in a golang shell
+          docker build -t ${runtime.images.default.image}:${runtime.images.default.tag} . -f custom/Dockerfile
         onChange:
         - main.go
 deployments:
diff --git a/examples/custom-builder/kube/deployment.yaml b/examples/custom-builder/kube/deployment.yaml
@@ -4,6 +4,10 @@ metadata:
   name: devspace
 spec:
   replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: devspace-default
+      app.kubernetes.io/name: devspace-app
   template:
     metadata:
       labels:
@@ -12,4 +16,5 @@ spec:
     spec:
       containers:
         - name: default
-          image: devspace
+          image: myusername/devspace
+          imagePullPolicy: Never
diff --git a/pkg/devspace/build/builder/custom/custom.go b/pkg/devspace/build/builder/custom/custom.go
@@ -2,6 +2,10 @@ package custom
 
 import (
 	"fmt"
+	"github.com/loft-sh/devspace/pkg/devspace/config"
+	"github.com/loft-sh/devspace/pkg/devspace/config/loader/variable/runtime"
+	"github.com/loft-sh/devspace/pkg/devspace/dependency/types"
+	"github.com/loft-sh/devspace/pkg/util/shell"
 	"io"
 	"path/filepath"
 	"strings"
@@ -28,14 +32,20 @@ type Builder struct {
 
 	imageConfigName string
 	imageTags       []string
+
+	config       config.Config
+	dependencies []types.Dependency
 }
 
 // NewBuilder creates a new custom builder
-func NewBuilder(imageConfigName string, imageConf *latest.ImageConfig, imageTags []string) *Builder {
+func NewBuilder(imageConfigName string, imageConf *latest.ImageConfig, imageTags []string, config config.Config, dependencies []types.Dependency) *Builder {
 	return &Builder{
 		imageConfigName: imageConfigName,
 		imageConf:       imageConf,
 		imageTags:       imageTags,
+
+		config:       config,
+		dependencies: dependencies,
 	}
 }
 
@@ -74,7 +84,7 @@ func (b *Builder) ShouldRebuild(cache *generated.CacheConfig, forceRebuild bool,
 	imageCache := cache.GetImageCache(b.imageConfigName)
 
 	// only rebuild Docker image when Dockerfile or context has changed since latest build
-	mustRebuild := imageCache.Tag == "" || imageCache.ImageConfigHash != imageConfigHash || imageCache.CustomFilesHash != customFilesHash
+	mustRebuild := forceRebuild || b.imageConf.RebuildStrategy == latest.RebuildStrategyAlways || imageCache.Tag == "" || imageCache.ImageConfigHash != imageConfigHash || imageCache.CustomFilesHash != customFilesHash
 
 	imageCache.ImageConfigHash = imageConfigHash
 	imageCache.CustomFilesHash = customFilesHash
@@ -87,8 +97,23 @@ func (b *Builder) Build(devspacePID string, log logpkg.Logger) error {
 	// Build arguments
 	args := []string{}
 
-	// add args
-	args = append(args, b.imageConf.Build.Custom.Args...)
+	// resolve command
+	if len(b.imageTags) > 0 {
+		key := fmt.Sprintf("images.%s", b.imageConfigName)
+		b.config.SetRuntimeVariable(key, b.imageConf.Image+":"+b.imageTags[0])
+		b.config.SetRuntimeVariable(key+".image", b.imageConf.Image)
+		b.config.SetRuntimeVariable(key+".tag", b.imageTags[0])
+	}
+	
+	// loop over args
+	for i := range b.imageConf.Build.Custom.Args {
+		resolvedArg, err := runtime.NewRuntimeResolver(false).FillRuntimeVariablesAsString(b.imageConf.Build.Custom.Args[i], b.config, b.dependencies)
+		if err != nil {
+			return err
+		}
+		
+		args = append(args, resolvedArg)
+	}
 
 	// add image arg
 	if !b.imageConf.Build.Custom.SkipImageArg {
@@ -106,7 +131,14 @@ func (b *Builder) Build(devspacePID string, log logpkg.Logger) error {
 	}
 
 	// append the rest
-	args = append(args, b.imageConf.Build.Custom.AppendArgs...)
+	for i := range b.imageConf.Build.Custom.AppendArgs {
+		resolvedArg, err := runtime.NewRuntimeResolver(false).FillRuntimeVariablesAsString(b.imageConf.Build.Custom.AppendArgs[i], b.config, b.dependencies)
+		if err != nil {
+			return err
+		}
+
+		args = append(args, resolvedArg)
+	}
 
 	// get the command
 	commandPath := b.imageConf.Build.Custom.Command
@@ -122,11 +154,11 @@ func (b *Builder) Build(devspacePID string, log logpkg.Logger) error {
 		return fmt.Errorf("no command specified for custom builder")
 	}
 
-	// make sure the path has the correct slashes
-	commandPath = filepath.FromSlash(commandPath)
-
-	// Create the command
-	cmd := command.NewStreamCommand(commandPath, args)
+	// resolve command and args
+	commandPath, err := runtime.NewRuntimeResolver(false).FillRuntimeVariablesAsString(commandPath, b.config, b.dependencies)
+	if err != nil {
+		return err
+	}
 
 	// Determine output writer
 	var writer io.Writer
@@ -138,9 +170,16 @@ func (b *Builder) Build(devspacePID string, log logpkg.Logger) error {
 
 	log.Infof("Build %s:%s with custom command '%s %s'", b.imageConf.Image, b.imageTags[0], commandPath, strings.Join(args, " "))
 
-	err := cmd.Run(writer, writer, nil)
-	if err != nil {
-		return errors.Errorf("Error building image: %v", err)
+	if len(args) == 0 {
+		err = shell.ExecuteShellCommand(commandPath, args, filepath.Dir(b.config.Path()), writer, writer, nil)
+		if err != nil {
+			return errors.Errorf("error building image: %v", err)
+		}
+	} else {
+		err = command.NewStreamCommand(commandPath, args).Run(writer, writer, nil)
+		if err != nil {
+			return errors.Errorf("error building image: %v", err)
+		}
 	}
 
 	log.Done("Done processing image '" + b.imageConf.Image + "'")
diff --git a/pkg/devspace/build/create_builder.go b/pkg/devspace/build/create_builder.go
@@ -24,7 +24,7 @@ func (c *controller) createBuilder(imageConfigName string, imageConf *latest.Ima
 	var builder builder.Interface
 
 	if imageConf.Build != nil && imageConf.Build.Custom != nil {
-		builder = custom.NewBuilder(imageConfigName, imageConf, imageTags)
+		builder = custom.NewBuilder(imageConfigName, imageConf, imageTags, c.config, c.dependencies)
 	} else if imageConf.Build != nil && imageConf.Build.BuildKit != nil {
 		log.StartWait("Creating BuildKit builder")
 		defer log.StopWait()
diff --git a/pkg/devspace/config/loader/parser.go b/pkg/devspace/config/loader/parser.go
@@ -2,6 +2,7 @@ package loader
 
 import (
 	"github.com/loft-sh/devspace/pkg/devspace/config/loader/variable"
+	"github.com/loft-sh/devspace/pkg/devspace/config/loader/variable/runtime"
 	"github.com/loft-sh/devspace/pkg/devspace/config/versions"
 	"github.com/loft-sh/devspace/pkg/devspace/config/versions/latest"
 	"github.com/loft-sh/devspace/pkg/util/log"
@@ -96,16 +97,7 @@ func (p *profilesParser) Parse(originalRawConfig map[interface{}]interface{}, ra
 
 func fillVariablesAndParse(resolver variable.Resolver, preparedConfig map[interface{}]interface{}, log log.Logger) (*latest.Config, error) {
 	// fill in variables and expressions (leave out
-	preparedConfigInterface, err := resolver.FillVariablesExclude(preparedConfig, []string{
-		"/deployments/*/helm/values/**",
-		"/hooks/*/command",
-		"/hooks/*/args/*",
-		"/hooks/*/container/imageSelector",
-		"/dev/ports/*/imageSelector",
-		"/dev/sync/*/imageSelector",
-		"/dev/logs/*/selectors/*/imageSelector",
-		"/dev/terminal/imageSelector",
-	})
+	preparedConfigInterface, err := resolver.FillVariablesExclude(preparedConfig, runtime.Locations)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/devspace/config/loader/variable/resolver.go b/pkg/devspace/config/loader/variable/resolver.go
@@ -3,6 +3,7 @@ package variable
 import (
 	"fmt"
 	"github.com/loft-sh/devspace/pkg/devspace/config/loader/variable/expression"
+	"github.com/loft-sh/devspace/pkg/devspace/config/loader/variable/runtime"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -66,7 +67,7 @@ func (r *resolver) fillVariables(haystack interface{}, exclude []*regexp.Regexp)
 				return nil, err
 			}
 		}
-		
+
 		return t, nil
 	}
 
@@ -120,6 +121,13 @@ func (r *resolver) FindVariables(haystack interface{}) (map[string]bool, error)
 		}
 	}
 
+	// filter out runtime environment variables
+	for k := range varsUsed {
+		if strings.HasPrefix(k, "runtime.") {
+			delete(varsUsed, k)
+		}
+	}
+
 	return varsUsed, nil
 }
 
@@ -396,7 +404,7 @@ func (r *resolver) fillVariable(name string, definition *latest.Variable) (inter
 
 	// is runtime variable
 	if strings.HasPrefix(name, "runtime.") {
-		return nil, fmt.Errorf("cannot resolve %s in this config area as this config region is loaded on startup. Please check the DevSpace docs in which config regions you can use runtime variables", name)
+		return nil, fmt.Errorf("cannot resolve %s in this config area as this config region is loaded on startup. You can only use runtime variables in the following locations: \n  %s", name, strings.Join(runtime.Locations, "\n  "))
 	}
 
 	// fill variable without definition
diff --git a/pkg/devspace/config/loader/variable/runtime/runtime_variable.go b/pkg/devspace/config/loader/variable/runtime/runtime_variable.go
@@ -7,6 +7,21 @@ import (
 	"strings"
 )
 
+var Locations = []string{
+	"/images/*/build/custom/command",
+	"/images/*/build/custom/commands/*/command",
+	"/images/*/build/custom/args/*",
+	"/images/*/build/custom/appendArgs/*",
+	"/deployments/*/helm/values/**",
+	"/hooks/*/command",
+	"/hooks/*/args/*",
+	"/hooks/*/container/imageSelector",
+	"/dev/ports/*/imageSelector",
+	"/dev/sync/*/imageSelector",
+	"/dev/logs/*/selectors/*/imageSelector",
+	"/dev/terminal/imageSelector",
+}
+
 // NewRuntimeVariable creates a new variable that is loaded during runtime
 func NewRuntimeVariable(name string, config config.Config, dependencies []types.Dependency, builtImages map[string]string) *runtimeVariable {
 	return &runtimeVariable{
@@ -57,6 +72,12 @@ func (e *runtimeVariable) Load() (bool, interface{}, error) {
 		return false, nil, fmt.Errorf("couldn't find runtime variable %s", e.name)
 	}
 
+	// generic retrieve runtime variable
+	out, ok := runtimeVariables[runtimeVar]
+	if ok {
+		return false, out, nil
+	}
+
 	// get image info from generated config
 	if strings.HasPrefix(runtimeVar, "images.") {
 		runtimeVar = strings.TrimPrefix(runtimeVar, "images.")
@@ -126,11 +147,5 @@ func (e *runtimeVariable) Load() (bool, interface{}, error) {
 		return false, nil, fmt.Errorf("couldn't find imageName %s resolving variable %s", imageName, e.name)
 	}
 
-	// generic retrieve runtime variable
-	out, ok := runtimeVariables[runtimeVar]
-	if !ok {
-		return false, nil, fmt.Errorf("couldn't find runtime variable %s", e.name)
-	}
-
-	return false, out, nil
+	return false, nil, fmt.Errorf("couldn't find runtime variable %s", e.name)
 }
diff --git a/pkg/devspace/config/versions/latest/schema.go b/pkg/devspace/config/versions/latest/schema.go
@@ -448,6 +448,7 @@ type BuildOptions struct {
 type DeploymentConfig struct {
 	Name      string         `yaml:"name" json:"name"`
 	Namespace string         `yaml:"namespace,omitempty" json:"namespace,omitempty"`
+	Disabled  bool           `yaml:"disabled,omitempty" json:"disabled,omitempty"`
 	Helm      *HelmConfig    `yaml:"helm,omitempty" json:"helm,omitempty"`
 	Kubectl   *KubectlConfig `yaml:"kubectl,omitempty" json:"kubectl,omitempty"`
 }
@@ -956,6 +957,7 @@ type PodPatch struct {
 type DependencyConfig struct {
 	Name                     string          `yaml:"name" json:"name"`
 	Source                   *SourceConfig   `yaml:"source" json:"source"`
+	Disabled                 bool            `yaml:"disabled,omitempty" json:"disabled,omitempty"`
 	Profile                  string          `yaml:"profile,omitempty" json:"profile,omitempty"`
 	Profiles                 []string        `yaml:"profiles,omitempty" json:"profiles,omitempty"`
 	ProfileParents           []string        `yaml:"profileParents,omitempty" json:"profileParents,omitempty"`
@@ -1012,6 +1014,9 @@ type HookConfig struct {
 	// Name is the name of the hook
 	Name string `yaml:"name,omitempty" json:"name,omitempty"`
 
+	// If true, the hook is disabled and not executed
+	Disabled bool `yaml:"disabled,omitempty" json:"disabled,omitempty"`
+
 	// Events are the events when the hook should be executed
 	Events []string `yaml:"events" json:"events"`
 
@@ -1212,6 +1217,9 @@ type PatchConfig struct {
 
 // PullSecretConfig defines a pull secret that should be created by DevSpace
 type PullSecretConfig struct {
+	// If true, the pull secret will be not created
+	Disabled bool `yaml:"disabled,omitempty" json:"disabled,omitempty"`
+
 	// The registry to create the image pull secret for.
 	// e.g. gcr.io
 	Registry string `yaml:"registry" json:"registry"`
diff --git a/pkg/devspace/dependency/resolver.go b/pkg/devspace/dependency/resolver.go
@@ -140,6 +140,11 @@ func (r *resolver) resolveRecursive(basePath, parentID string, currentDependency
 		currentDependency.children = []types.Dependency{}
 	}
 	for _, dependencyConfig := range dependencies {
+		if dependencyConfig.Disabled {
+			r.log.Debugf("Skip dependency %s, because it is disabled", dependencyConfig.Name)
+			continue
+		}
+
 		ID, err := util.GetDependencyID(basePath, dependencyConfig)
 		if err != nil {
 			return err
diff --git a/pkg/devspace/deploy/deploy.go b/pkg/devspace/deploy/deploy.go
@@ -63,6 +63,11 @@ func (c *controller) Render(options *Options, out io.Writer, log log.Logger) err
 		}
 
 		for _, deployConfig := range config.Deployments {
+			if deployConfig.Disabled {
+				log.Debugf("Skip deployment %s, because it is disabled", deployConfig.Name)
+				continue
+			}
+
 			if len(options.Deployments) > 0 {
 				shouldSkip := true
 
@@ -164,6 +169,11 @@ func (c *controller) Deploy(options *Options, log log.Logger) error {
 		}
 
 		for _, deployConfig := range config.Deployments {
+			if deployConfig.Disabled {
+				log.Debugf("Skip deployment %s, because it is disabled", deployConfig.Name)
+				continue
+			}
+
 			if len(options.Deployments) > 0 {
 				shouldSkip := true
 
@@ -290,6 +300,10 @@ func (c *controller) Purge(deployments []string, log log.Logger) error {
 				deployClient deployer.Interface
 				deployConfig = config.Deployments[i]
 			)
+			if deployConfig.Disabled {
+				log.Debugf("Skip deployment %s, because it is disabled", deployConfig.Name)
+				continue
+			}
 
 			// Check if we should skip deleting deployment
 			if deployments != nil {
diff --git a/pkg/devspace/hook/hook.go b/pkg/devspace/hook/hook.go
@@ -121,7 +121,7 @@ func executeSingle(client kubectl.Client, config config.Config, dependencies []t
 
 		// Execute hooks
 		for _, hookConfig := range hooksToExecute {
-			if !command.ShouldExecuteOnOS(hookConfig.OperatingSystem) {
+			if hookConfig.Disabled || !command.ShouldExecuteOnOS(hookConfig.OperatingSystem) {
 				continue
 			}
 
diff --git a/pkg/devspace/pullsecrets/init.go b/pkg/devspace/pullsecrets/init.go
@@ -21,7 +21,13 @@ func (r *client) CreatePullSecrets() (err error) {
 
 	// gather pull secrets from pullSecrets
 	if r.config != nil {
-		createPullSecrets = append(createPullSecrets, r.config.Config().PullSecrets...)
+		for _, pullSecret := range r.config.Config().PullSecrets {
+			if pullSecret.Disabled {
+				continue
+			}
+
+			createPullSecrets = append(createPullSecrets, pullSecret)
+		}
 
 		// gather pull secrets from images
 		for _, imageConf := range r.config.Config().Images {

Original file line number	Diff line number	Diff line change
`@@ -3,6 +3,7 @@ package variable`
`3`	`3`	`import (`
`4`	`4`	`"fmt"`
`5`	`5`	`"github.com/loft-sh/devspace/pkg/devspace/config/loader/variable/expression"`
	`6`	`+ "github.com/loft-sh/devspace/pkg/devspace/config/loader/variable/runtime"`
`6`	`7`	`"path/filepath"`
`7`	`8`	`"regexp"`
`8`	`9`	`"strings"`
`@@ -66,7 +67,7 @@ func (r resolver) fillVariables(haystack interface{}, exclude []regexp.Regexp)`
`66`	`67`	`return nil, err`
`67`	`68`	`}`
`68`	`69`	`}`
`69`		`-`
	`70`	`+`
`70`	`71`	`return t, nil`
`71`	`72`	`}`
`72`	`73`
`@@ -120,6 +121,13 @@ func (r *resolver) FindVariables(haystack interface{}) (map[string]bool, error)`
`120`	`121`	`}`
`121`	`122`	`}`
`122`	`123`
	`124`	`+ // filter out runtime environment variables`
	`125`	`+ for k := range varsUsed {`
	`126`	`+ if strings.HasPrefix(k, "runtime.") {`
	`127`	`+ delete(varsUsed, k)`
	`128`	`+ }`
	`129`	`+ }`
	`130`	`+`
`123`	`131`	`return varsUsed, nil`
`124`	`132`	`}`
`125`	`133`
`@@ -396,7 +404,7 @@ func (r resolver) fillVariable(name string, definition latest.Variable) (inter`
`396`	`404`
`397`	`405`	`// is runtime variable`
`398`	`406`	`if strings.HasPrefix(name, "runtime.") {`
`399`		`- return nil, fmt.Errorf("cannot resolve %s in this config area as this config region is loaded on startup. Please check the DevSpace docs in which config regions you can use runtime variables", name)`
	`407`	`+ return nil, fmt.Errorf("cannot resolve %s in this config area as this config region is loaded on startup. You can only use runtime variables in the following locations: \n %s", name, strings.Join(runtime.Locations, "\n "))`
`400`	`408`	`}`
`401`	`409`
`402`	`410`	`// fill variable without definition`