Merge pull request #1259 from merico-dev/feat-app-spec-option

feat: app support ci/cd default config for app.spec

Author: daniel-hutao

internal/pkg/configmanager/app.go

@@ -10,25 +10,30 @@ const (
 	repoScaffoldingPluginName = "repo-scaffolding"
 )
 
+type repoTemplate struct {
+	*scm.SCMInfo `yaml:",inline"`
+	Vars         map[string]any `yaml:"vars"`
+}
+
 type app struct {
-	Name         string         `yaml:"name" mapstructure:"name"`
-	Spec         map[string]any `yaml:"spec" mapstructure:"spec"`
-	Repo         *scm.SCMInfo   `yaml:"repo" mapstructure:"repo"`
-	RepoTemplate *scm.SCMInfo   `yaml:"repoTemplate" mapstructure:"repoTemplate"`
-	CIRawConfigs []pipelineRaw  `yaml:"ci" mapstructure:"ci"`
-	CDRawConfigs []pipelineRaw  `yaml:"cd" mapstructure:"cd"`
+	Name         string        `yaml:"name" mapstructure:"name"`
+	Spec         *appSpec      `yaml:"spec" mapstructure:"spec"`
+	Repo         *scm.SCMInfo  `yaml:"repo" mapstructure:"repo"`
+	RepoTemplate *repoTemplate `yaml:"repoTemplate" mapstructure:"repoTemplate"`
+	CIRawConfigs []pipelineRaw `yaml:"ci" mapstructure:"ci"`
+	CDRawConfigs []pipelineRaw `yaml:"cd" mapstructure:"cd"`
 }
 
 // getAppPipelineTool generate ci/cd tools from app config
 func (a *app) generateCICDToolsFromAppConfig(templateMap map[string]string, appVars map[string]any) (Tools, error) {
 	allPipelineRaw := append(a.CIRawConfigs, a.CDRawConfigs...)
 	var tools Tools
 	for _, p := range allPipelineRaw {
-		t, err := p.newPipeline(a.Repo, templateMap, appVars)
+		t, err := p.getPipelineTemplate(templateMap, appVars)
 		if err != nil {
 			return nil, err
 		}
-		pipelineTool, err := t.getPipelineTool(a.Name)
+		pipelineTool, err := t.generatePipelineTool(a)
 		if err != nil {
 			return nil, err
 		}
@@ -39,7 +44,7 @@ func (a *app) generateCICDToolsFromAppConfig(templateMap map[string]string, appV
 }
 
 // getRepoTemplateTool will use repo-scaffolding plugin for app
-func (a *app) getRepoTemplateTool(appVars map[string]any) (*Tool, error) {
+func (a *app) getRepoTemplateTool() (*Tool, error) {
 	if a.Repo == nil {
 		return nil, fmt.Errorf("app.repo field can't be empty")
 	}
@@ -56,7 +61,7 @@ func (a *app) getRepoTemplateTool(appVars map[string]any) (*Tool, error) {
 			repoScaffoldingPluginName, a.Name, RawOptions{
 				"destinationRepo": RawOptions(appRepo.Encode()),
 				"sourceRepo":      RawOptions(templateRepo.Encode()),
-				"vars":            RawOptions(appVars),
+				"vars":            RawOptions(a.RepoTemplate.Vars),
 			},
 		), nil
 	}

internal/pkg/configmanager/appspec.go

@@ -0,0 +1,42 @@
+package configmanager
+
+import (
+	"github.com/imdario/mergo"
+
+	"github.com/devstream-io/devstream/pkg/util/log"
+	"github.com/devstream-io/devstream/pkg/util/mapz"
+)
+
+// appSpec is app special options
+type appSpec struct {
+	// language config
+	Language  string `yaml:"language" mapstructure:"language"`
+	FrameWork string `yaml:"framework" mapstructure:"framework"`
+}
+
+// merge will merge vars and appSpec
+func (s *appSpec) merge(vars map[string]any) map[string]any {
+	specMap, err := mapz.DecodeStructToMap(s)
+	if err != nil {
+		log.Warnf("appspec %+v decode failed: %+v", s, err)
+		return map[string]any{}
+	}
+	if err := mergo.Merge(&specMap, vars); err != nil {
+		log.Warnf("appSpec %+v merge map failed: %+v", s, err)
+		return vars
+	}
+	return specMap
+}
+
+func (s *appSpec) updatePiplineOption(options RawOptions) {
+	if _, exist := options["language"]; !exist && s.hasLanguageConfig() {
+		options["language"] = RawOptions{
+			"name":      s.Language,
+			"framework": s.FrameWork,
+		}
+	}
+}
+
+func (s *appSpec) hasLanguageConfig() bool {
+	return s.Language != "" || s.FrameWork != ""
+}

internal/pkg/configmanager/config.go

@@ -6,7 +6,6 @@ import (
 	"gopkg.in/yaml.v3"
 
 	"github.com/devstream-io/devstream/pkg/util/log"
-	"github.com/devstream-io/devstream/pkg/util/mapz"
 )
 
 // Config is a general config in DevStream.
@@ -55,16 +54,15 @@ func (c *Config) getToolsWithVarsFromApp(a app) (Tools, error) {
 		return nil, fmt.Errorf("app parse yaml failed: %w", err)
 	}
 
-	rawApp.setDefault()
-	appVars := mapz.Merge(c.Vars, rawApp.Spec)
-
 	// 3. generate app repo and template repo from scmInfo
-	repoScaffoldingTool, err := rawApp.getRepoTemplateTool(appVars)
+	rawApp.setDefault()
+	repoScaffoldingTool, err := rawApp.getRepoTemplateTool()
 	if err != nil {
 		return nil, fmt.Errorf("app[%s] get repo failed: %w", rawApp.Name, err)
 	}
 
 	// 4. get ci/cd pipelineTemplates
+	appVars := rawApp.Spec.merge(c.Vars)
 	tools, err := rawApp.generateCICDToolsFromAppConfig(c.pipelineTemplateMap, appVars)
 	if err != nil {
 		return nil, fmt.Errorf("app[%s] get pipeline tools failed: %w", rawApp.Name, err)

internal/pkg/configmanager/configmanager.go

@@ -74,6 +74,6 @@ func (m *Manager) getConfigFromFile() (*Config, error) {
 
 // escapeBrackets is used to escape []byte(": [[xxx]]xxx\n") to []byte(": \"[[xxx]]\"xxx\n")
 func escapeBrackets(param []byte) []byte {
-	re := regexp.MustCompile(`([^:]+:)(\s*)(\[\[[^\]]+\]\][^\s]*)`)
+	re := regexp.MustCompile(`([^:]+:)(\s*)((\[\[[^\]]+\]\][^\s\[]*)+)[^\s#\n]*`)
 	return re.ReplaceAll(param, []byte("$1$2\"$3\""))
 }

internal/pkg/configmanager/configmanager_test.go

@@ -86,7 +86,7 @@ pipelineTemplates:
       namespace: [[ argocdNamespace ]] # you can use global vars in templates
     destination:
       server: https://kubernetes.default.svc
-      namespace: default
+      namespace: devstream-io
     source:
       valuefile: values.yaml
       path: helm/[[ app ]]
@@ -128,6 +128,10 @@ var _ = Describe("LoadConfig", func() {
 		Options: RawOptions{
 			"instanceID": "service-a",
 			"pipeline": RawOptions{
+				"language": RawOptions{
+					"name":      "python",
+					"framework": "django",
+				},
 				"docker": RawOptions{
 					"registry": RawOptions{
 						"repository": "service-a",
@@ -159,28 +163,17 @@ var _ = Describe("LoadConfig", func() {
 		},
 		Options: RawOptions{
 			"instanceID": "service-a",
-			"pipeline": RawOptions{
-				"destination": RawOptions{
-					"namespace": "devstream-io",
-					"server":    "https://kubernetes.default.svc",
-				},
-				"app": RawOptions{
-					"namespace": "argocd",
-				},
-				"source": RawOptions{
-					"valuefile": "values.yaml",
-					"path":      "helm/service-a",
-					"repoURL":   "${{repo-scaffolding.myapp.outputs.repoURL}}",
-				},
-				"configLocation": "",
+			"destination": RawOptions{
+				"namespace": "devstream-io",
+				"server":    "https://kubernetes.default.svc",
 			},
-			"scm": RawOptions{
-				"url":     "https://github.com/devstream-io/service-a",
-				"apiURL":  "gitlab.com/some/path/to/your/api",
-				"owner":   "devstream-io",
-				"org":     "devstream-io",
-				"name":    "service-a",
-				"scmType": "github",
+			"app": RawOptions{
+				"namespace": "argocd",
+			},
+			"source": RawOptions{
+				"valuefile": "values.yaml",
+				"path":      "helm/service-a",
+				"repoURL":   "${{repo-scaffolding.myapp.outputs.repoURL}}",
 			},
 		},
 	}
@@ -210,14 +203,7 @@ var _ = Describe("LoadConfig", func() {
 				"repo":     "dtm-scaffolding-golang",
 				"branch":   "main",
 			},
-			"vars": RawOptions{
-				"foo1":            "bar1",
-				"foo2":            "bar2",
-				"registryType":    "dockerhub",
-				"framework":       "django",
-				"language":        "python",
-				"argocdNamespace": "argocd",
-			},
+			"vars": RawOptions{},
 		},
 	}
 
@@ -273,17 +259,20 @@ var _ = Describe("LoadConfig", func() {
 var _ = Describe("escapeBrackets", func() {
 	When("escape brackets", func() {
 		It("should works right", func() {
-			testStrBytes1 := "foo: [[ foo ]]\n"
-			testStr2 := "foo: xx[[ foo ]]\n"
-			testStr3 := "foo: [[ foo ]]xx\n"
-
-			retStr1 := escapeBrackets([]byte(testStrBytes1))
-			retStr2 := escapeBrackets([]byte(testStr2))
-			retStr3 := escapeBrackets([]byte(testStr3))
+			testMap := map[string]string{
+				"foo: [[ foo ]]":           "foo: \"[[ foo ]]\"",
+				"foo: [[ foo ]] #comment":  "foo: \"[[ foo ]]\" #comment",
+				"foo: xx[[ foo ]]":         "foo: xx[[ foo ]]",
+				"foo: [[ foo ]]xx":         "foo: \"[[ foo ]]xx\"",
+				"foo: [[ foo ]]/[[ poo ]]": "foo: \"[[ foo ]]/[[ poo ]]\"",
+				`foo: [[ test ]]
+poo: [[ gg ]]`: "foo: \"[[ test ]]\"\npoo: \"[[ gg ]]\"",
+			}
 
-			Expect(string(retStr1)).To(Equal("foo: \"[[ foo ]]\"\n"))
-			Expect(string(retStr2)).To(Equal("foo: xx[[ foo ]]\n"))
-			Expect(string(retStr3)).To(Equal("foo: \"[[ foo ]]xx\"\n"))
+			for testStr, expectStr := range testMap {
+				retStr1 := escapeBrackets([]byte(testStr))
+				Expect(string(retStr1)).Should(Equal(expectStr))
+			}
 		})
 	})
 })

internal/pkg/configmanager/pipelineDefault.go

@@ -0,0 +1,96 @@
+package configmanager
+
+import (
+	"fmt"
+
+	"github.com/devstream-io/devstream/pkg/util/log"
+)
+
+var optionConfiguratorMap = map[string]pipelineOption{
+	"github-actions":   githubGeneral,
+	"gitlab-ci":        gitlabGeneral,
+	"jenkins-pipeline": jenkinsGeneral,
+	"argocdapp":        argocdApp,
+}
+
+type pipelineOptionGenerator func(originOption RawOptions, app *app) RawOptions
+
+type pipelineOption struct {
+	defaultConfigLocation string
+	optionGeneratorFunc   pipelineOptionGenerator
+}
+
+// TODO(steinliber) unify all ci/cd config to same config options
+var (
+	// github actions pipeline options
+	githubGeneral = pipelineOption{
+		defaultConfigLocation: "[email protected]:devstream-io/ci-template.git//github-actions",
+		optionGeneratorFunc:   pipelineGeneralGenerator,
+	}
+	gitlabGeneral = pipelineOption{
+		defaultConfigLocation: "https://raw.githubusercontent.com/devstream-io/ci-template/main/gitlab-ci/.gitlab-ci.yml",
+		optionGeneratorFunc:   pipelineGeneralGenerator,
+	}
+	jenkinsGeneral = pipelineOption{
+		defaultConfigLocation: "https://raw.githubusercontent.com/devstream-io/ci-template/main/jenkins-pipeline/general/Jenkinsfile",
+		optionGeneratorFunc:   pipelineGeneralGenerator,
+	}
+	argocdApp = pipelineOption{
+		optionGeneratorFunc: pipelineArgocdAppGenerator,
+	}
+)
+
+// pipelineGeneralGenerator generate pipeline general options from RawOptions
+func pipelineGeneralGenerator(options RawOptions, app *app) RawOptions {
+	if app.Spec != nil {
+		app.Spec.updatePiplineOption(options)
+	}
+	// update image related config
+	newOption := make(RawOptions)
+	newOption["pipeline"] = options
+	newOption["scm"] = RawOptions(app.Repo.Encode())
+	return newOption
+}
+
+// pipelineArgocdAppGenerator generate argocdApp options from RawOptions
+func pipelineArgocdAppGenerator(options RawOptions, app *app) RawOptions {
+	// config app default options
+	if _, exist := options["app"]; !exist {
+		options["app"] = RawOptions{
+			"name":      app.Name,
+			"namespace": "argocd",
+		}
+	}
+	// config destination options
+	if _, exist := options["destination"]; !exist {
+		options["destination"] = RawOptions{
+			"server":    "https://kubernetes.default.svc",
+			"namespace": "default",
+		}
+	}
+	// config source default options
+	repoInfo, err := app.Repo.BuildRepoInfo()
+	if err != nil {
+		log.Errorf("parse argocd repoInfo failed: %+v", err)
+		return options
+	}
+	if source, sourceExist := options["source"]; sourceExist {
+		sourceMap := source.(RawOptions)
+		if _, repoURLExist := sourceMap["repoURL"]; !repoURLExist {
+			sourceMap["repoURL"] = repoInfo.CloneURL
+		}
+		options["source"] = sourceMap
+	} else {
+		options["source"] = RawOptions{
+			"valuefile": "values.yaml",
+			"path":      fmt.Sprintf("helm/%s", app.Name),
+			"repoURL":   repoInfo.CloneURL,
+		}
+	}
+	return options
+}
+
+// hasDefaultConfig check whether
+func (o *pipelineOption) hasDefaultConfig() bool {
+	return o.defaultConfigLocation != ""
+}