Merge pull request #1315 from merico-dev/feat-apps-cicd-pass-vars

feat: pass variable from ci to cd

Author: daniel-hutao

docs/plugins/argocdapp.md

@@ -72,3 +72,32 @@ In the example above:
 - We used `repo-scaffolding.golang-github`'s output as input for the `githubactions-golang` plugin.
 
 Pay attention to the `${{ xxx }}` part in the example. `${{ TOOL_NAME.PLUGIN.outputs.var}}` is the syntax for using an output.
+
+## Automatically Create Helm Configuration
+
+This plugin can push helm configuration automatically when your source.path helm config not exist, so you can use this plugin with helm configured alreay. For example:
+
+```yaml
+---
+tools:
+- name: go-webapp-argocd-deploy
+  plugin: argocdapp
+  dependsOn: ["repo-scaffolding.golang-github"]
+  options:
+    app:
+      name: hello
+      namespace: argocd
+    destination:
+      server: https://kubernetes.default.svc
+      namespace: default
+    source:
+      valuefile: values.yaml
+      path: charts/go-hello-http
+      repoURL: ${{repo-scaffolding.golang-github.outputs.repoURL}}
+    imageRepo:
+      url: http://test.barbor.com/library
+      user: test_owner
+      tag: "1.0.0"
+```
+
+This config will push default helm config to repo `${{repo-scaffolding.golang-github.outputs.repoURL}}`, and the generated config will use image `http://test.barbor.com/library/test_owner/hello:1.0.0` as inital image for helm.

internal/pkg/configmanager/app.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/devstream-io/devstream/pkg/util/log"
 	"github.com/devstream-io/devstream/pkg/util/scm"
+	"github.com/devstream-io/devstream/pkg/util/scm/git"
 )
 
 const (
@@ -23,41 +24,49 @@ type app struct {
 	RepoTemplate *repoTemplate `yaml:"repoTemplate" mapstructure:"repoTemplate"`
 	CIRawConfigs []pipelineRaw `yaml:"ci" mapstructure:"ci"`
 	CDRawConfigs []pipelineRaw `yaml:"cd" mapstructure:"cd"`
+
+	// these two variables is used internal for convince
+	// repoInfo is generated from Repo field with setDefault method
+	repoInfo *git.RepoInfo `yaml:"-" mapstructure:"-"`
+	// repoTemplateInfo is generated from RepoTemplate field with setDefault method
+	repoTemplateInfo *git.RepoInfo `yaml:"-" mapstructure:"-"`
 }
 
 func (a *app) getTools(vars map[string]any, templateMap map[string]string) (Tools, error) {
-	// generate app repo and template repo from scmInfo
-	a.setDefault()
-	repoScaffoldingTool, err := a.getRepoTemplateTool()
-	if err != nil {
-		return nil, fmt.Errorf("app[%s] can't get valid repo config: %w", a.Name, err)
+	// 1. set app default field repoInfo and repoTemplateInfo
+	if err := a.setDefault(); err != nil {
+		return nil, err
 	}
 
-	// get ci/cd pipelineTemplates
+	// 2. get ci/cd pipelineTemplates
 	appVars := a.Spec.merge(vars)
 	tools, err := a.generateCICDTools(templateMap, appVars)
 	if err != nil {
 		return nil, fmt.Errorf("app[%s] get pipeline tools failed: %w", a.Name, err)
 	}
+
+	// 3. generate app repo and template repo from scmInfo
+	repoScaffoldingTool := a.generateRepoTemplateTool()
 	if repoScaffoldingTool != nil {
 		tools = append(tools, repoScaffoldingTool)
 	}
-
 	log.Debugf("Have got %d tools from app %s.", len(tools), a.Name)
-
 	return tools, nil
 }
 
-// getAppPipelineTool generate ci/cd tools from app config
+// generateAppPipelineTool generate ci/cd tools from app config
 func (a *app) generateCICDTools(templateMap map[string]string, appVars map[string]any) (Tools, error) {
 	allPipelineRaw := append(a.CIRawConfigs, a.CDRawConfigs...)
 	var tools Tools
+	// pipelineGlobalVars is used to pass variable from ci/cd pipelines
+	pipelineGlobalVars := a.newPipelineGlobalOptionFromApp()
 	for _, p := range allPipelineRaw {
 		t, err := p.getPipelineTemplate(templateMap, appVars)
 		if err != nil {
 			return nil, err
 		}
-		pipelineTool, err := t.generatePipelineTool(a)
+		t.updatePipelineVars(pipelineGlobalVars)
+		pipelineTool, err := t.generatePipelineTool(pipelineGlobalVars)
 		if err != nil {
 			return nil, err
 		}
@@ -67,41 +76,48 @@ func (a *app) generateCICDTools(templateMap map[string]string, appVars map[strin
 	return tools, nil
 }
 
-// getRepoTemplateTool will use repo-scaffolding plugin for app
-func (a *app) getRepoTemplateTool() (*Tool, error) {
-	if a.Repo == nil {
-		return nil, fmt.Errorf("app.repo field can't be empty")
-	}
-	appRepo, err := a.Repo.BuildRepoInfo()
-	if err != nil {
-		return nil, fmt.Errorf("configmanager[app] parse repo failed: %w", err)
-	}
-	if a.RepoTemplate != nil {
+// generateRepoTemplateTool will use repo-scaffolding plugin for app
+func (a *app) generateRepoTemplateTool() *Tool {
+	if a.repoTemplateInfo != nil {
+		templateVars := make(RawOptions)
 		// templateRepo doesn't need auth info
-		templateRepo, err := a.RepoTemplate.BuildRepoInfo()
-		templateRepo.NeedAuth = false
-		if err != nil {
-			return nil, fmt.Errorf("configmanager[app] parse repoTemplate failed: %w", err)
-		}
 		if a.RepoTemplate.Vars == nil {
-			a.RepoTemplate.Vars = make(RawOptions)
+			templateVars = make(RawOptions)
 		}
 		return newTool(
 			repoScaffoldingPluginName, a.Name, RawOptions{
-				"destinationRepo": RawOptions(appRepo.Encode()),
-				"sourceRepo":      RawOptions(templateRepo.Encode()),
-				"vars":            a.RepoTemplate.Vars,
+				"destinationRepo": RawOptions(a.repoInfo.Encode()),
+				"sourceRepo":      RawOptions(a.repoTemplateInfo.Encode()),
+				"vars":            templateVars,
 			},
-		), nil
+		)
 	}
-	return nil, nil
+	return nil
 }
 
 // setDefault will set repoName to appName if repo.name field is empty
-func (a *app) setDefault() {
-	if a.Repo != nil && a.Repo.Name == "" {
+func (a *app) setDefault() error {
+	if a.Repo == nil {
+		return fmt.Errorf("configmanager[app] is invalid, repo field must be configured")
+	}
+	if a.Repo.Name == "" {
 		a.Repo.Name = a.Name
 	}
+	appRepo, err := a.Repo.BuildRepoInfo()
+	if err != nil {
+		return fmt.Errorf("configmanager[app] parse repo failed: %w", err)
+	}
+	a.repoInfo = appRepo
+	if a.RepoTemplate != nil {
+		// templateRepo doesn't need auth info
+		templateRepo, err := a.RepoTemplate.BuildRepoInfo()
+		if err != nil {
+			return fmt.Errorf("configmanager[app] parse repoTemplate failed: %w", err)
+		}
+		templateRepo.NeedAuth = false
+		a.repoTemplateInfo = templateRepo
+	}
+	return nil
 }
 
 // since all plugin depends on code is deployed, get dependsOn for repoTemplate
@@ -113,3 +129,13 @@ func (a *app) getRepoTemplateDependants() []string {
 	}
 	return dependsOn
 }
+
+// newPipelineGlobalOptionFromApp generate pipeline options used for pipeline option configuration
+func (a *app) newPipelineGlobalOptionFromApp() *pipelineGlobalOption {
+	return &pipelineGlobalOption{
+		RepoInfo: a.repoInfo,
+		AppSpec:  a.Spec,
+		Scm:      a.Repo,
+		AppName:  a.Name,
+	}
+}

internal/pkg/configmanager/app_test.go

@@ -1,8 +1,6 @@
 package configmanager
 
 import (
-	"fmt"
-
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
@@ -29,7 +27,7 @@ var _ = Describe("app struct", func() {
 			It("should return error", func() {
 				_, err := a.getTools(vars, templateMap)
 				Expect(err).Should(HaveOccurred())
-				Expect(err.Error()).Should(ContainSubstring(fmt.Sprintf("app[%s] can't get valid repo config", appName)))
+				Expect(err.Error()).Should(ContainSubstring("configmanager[app] is invalid, repo field must be configured"))
 			})
 		})
 		When("ci/cd template is not valid", func() {
@@ -52,6 +50,57 @@ var _ = Describe("app struct", func() {
 				Expect(err.Error()).Should(ContainSubstring("not found in pipelineTemplates"))
 			})
 		})
+		When("app repo template is empty", func() {
+			BeforeEach(func() {
+				a = &app{
+					Name: appName,
+					Repo: &scm.SCMInfo{
+						CloneURL: "http://test.com/test/test_app",
+					},
+				}
+			})
+			It("should return empty tools", func() {
+				tools, err := a.getTools(vars, templateMap)
+				Expect(err).ShouldNot(HaveOccurred())
+				Expect(len(tools)).Should(Equal(0))
+			})
+		})
+		When("repo url is not valid", func() {
+			BeforeEach(func() {
+				a = &app{
+					Name: appName,
+					Repo: &scm.SCMInfo{
+						CloneURL: "not_valid_url",
+					},
+				}
+			})
+			It("should return empty tools", func() {
+				_, err := a.getTools(vars, templateMap)
+				Expect(err).Should(HaveOccurred())
+				Expect(err.Error()).Should(ContainSubstring("configmanager[app] parse repo failed"))
+			})
+		})
+		When("template repo url is not valid", func() {
+			BeforeEach(func() {
+				a = &app{
+					Name: appName,
+					RepoTemplate: &repoTemplate{
+						SCMInfo: &scm.SCMInfo{
+							CloneURL: "not_valid_url",
+						},
+					},
+					Repo: &scm.SCMInfo{
+						CloneURL: "http://test.com/test/test_app",
+					},
+				}
+			})
+			It("should return empty tools", func() {
+				_, err := a.getTools(vars, templateMap)
+				Expect(err).Should(HaveOccurred())
+				Expect(err.Error()).Should(ContainSubstring("configmanager[app] parse repoTemplate failed"))
+			})
+		})
+
 	})
 
 	Context("generateCICDTools method", func() {

internal/pkg/configmanager/appspec.go

@@ -21,10 +21,7 @@ func (s *appSpec) merge(vars map[string]any) map[string]any {
 		log.Warnf("appspec %+v decode failed: %+v", s, err)
 		return map[string]any{}
 	}
-	if err := mergo.Merge(&specMap, vars); err != nil {
-		log.Warnf("appSpec %+v merge map failed: %+v", s, err)
-		return vars
-	}
+	_ = mergo.Merge(&specMap, vars)
 	return specMap
 }
 

internal/pkg/configmanager/config.go

@@ -2,8 +2,6 @@ package configmanager
 
 import (
 	"fmt"
-
-	"gopkg.in/yaml.v3"
 )
 
 // Config is a general config in DevStream.
@@ -24,19 +22,11 @@ func (c *Config) renderInstanceIDtoOptions() {
 
 func (c *Config) validate() error {
 	if c.Config.State == nil {
-		return fmt.Errorf("state is not defined")
+		return fmt.Errorf("config.state is not defined")
 	}
 
 	if err := c.Tools.validateAll(); err != nil {
 		return err
 	}
 	return nil
 }
-
-func (c *Config) String() string {
-	bs, err := yaml.Marshal(c)
-	if err != nil {
-		return err.Error()
-	}
-	return string(bs)
-}

internal/pkg/configmanager/config_test.go

@@ -1 +1,47 @@
 package configmanager
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Config struct", func() {
+	var (
+		c                    *Config
+		toolName, instanceID string
+	)
+	BeforeEach(func() {
+		c = &Config{}
+		toolName = "test_tool"
+		instanceID = "test_instance"
+	})
+	Context("renderInstanceIDtoOptions method", func() {
+		When("tool option is null", func() {
+			BeforeEach(func() {
+				c.Tools = Tools{
+					{Name: toolName, InstanceID: instanceID},
+				}
+			})
+			It("should set nil to RawOptions", func() {
+				c.renderInstanceIDtoOptions()
+				Expect(len(c.Tools)).Should(Equal(1))
+				tool := c.Tools[0]
+				Expect(tool.Options).Should(Equal(RawOptions{
+					"instanceID": instanceID,
+				}))
+			})
+		})
+	})
+	Context("validate method", func() {
+		When("config state is null", func() {
+			BeforeEach(func() {
+				c.Config.State = nil
+			})
+			It("should return err", func() {
+				err := c.validate()
+				Expect(err).Should(HaveOccurred())
+				Expect(err.Error()).Should(Equal("config.state is not defined"))
+			})
+		})
+	})
+})