feat: support nested vars

Signed-off-by: Bird <[email protected]>

Author: aFlyBird0

go.mod

@@ -171,6 +171,7 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
@@ -222,6 +223,7 @@ require (
 	gopkg.in/gorp.v1 v1.7.2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.62.0 // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	k8s.io/apiextensions-apiserver v0.22.4 // indirect

internal/pkg/configmanager/configmanager_test.go

@@ -22,6 +22,7 @@ config:
 vars:
   foo1: bar1
   foo2: 123
+  foo3: foo1+foo2=[[foo1]][[foo2 ]]!
   appName: service-a
   registryType: dockerhub
   argocdNamespace: argocd
@@ -68,6 +69,7 @@ tools:
   dependsOn: []
   options:
     foo1: [[ foo1 ]]
+    foo3: [[ foo3 ]]
 - name: plugin2
   instanceID: tluafed
   dependsOn: []
@@ -106,6 +108,7 @@ pipelineTemplates:
 			DependsOn:  []string{},
 			Options: RawOptions{
 				"foo1":       "bar1",
+				"foo3":       "foo1+foo2=bar1123!",
 				"instanceID": "default",
 			},
 		}
@@ -233,8 +236,10 @@ pipelineTemplates:
 				}))
 
 				// vars
-				Expect(len(cfg.Vars)).To(Equal(5))
+				Expect(len(cfg.Vars)).To(Equal(6))
 				Expect(cfg.Vars["foo1"]).To(Equal("bar1"))
+				Expect(cfg.Vars["foo2"]).To(Equal(123))
+				Expect(cfg.Vars["foo3"]).To(Equal("foo1+foo2=bar1123!"))
 
 				// tools
 				Expect(len(cfg.Tools)).To(Equal(5))

internal/pkg/configmanager/rawconfig.go

@@ -4,12 +4,13 @@ import (
 	"bytes"
 	"fmt"
 	"regexp"
+	"strings"
 	"unicode"
 
 	"gopkg.in/yaml.v3"
 )
 
-// rawConfig respent every valid config block for devstream
+// rawConfig represent every valid config block for devstream
 type rawConfig struct {
 	apps              []byte
 	pipelineTemplates []byte
@@ -77,9 +78,40 @@ func (c *rawConfig) validate() error {
 
 // getVars will generate variables from vars config
 func (c *rawConfig) getVars() (map[string]any, error) {
-	var globalVars map[string]any
-	err := yaml.Unmarshal(c.vars, &globalVars)
-	return globalVars, err
+	var (
+		globalVarsOrigin, globalVarsParsedNest map[string]any
+		err                                    error
+	)
+
+	// [[]] is array in yaml, replace them, or yaml.Unmarshal will return err
+	const (
+		leftOrigin, leftReplaced   = "[[", "【$【"
+		rightOrigin, rightReplaced = "]]", "】$】"
+	)
+	converter := strings.NewReplacer(leftOrigin, leftReplaced, rightOrigin, rightReplaced)
+	varsReplacedSquareBrackets := converter.Replace(string(c.vars))
+
+	// get origin vars from config
+	if err = yaml.Unmarshal([]byte(varsReplacedSquareBrackets), &globalVarsOrigin); err != nil {
+		return nil, err
+	}
+
+	// restore vars
+	restorer := strings.NewReplacer(leftReplaced, leftOrigin, rightReplaced, rightOrigin)
+	for k, v := range globalVarsOrigin {
+		switch value := v.(type) {
+		case string:
+			globalVarsOrigin[k] = restorer.Replace(value)
+		case []byte:
+			globalVarsOrigin[k] = restorer.Replace(string(value))
+		}
+	}
+
+	// parse nested vars
+	if globalVarsParsedNest, err = parseNestedVars(globalVarsOrigin); err != nil {
+		return nil, err
+	}
+	return globalVarsParsedNest, nil
 }
 
 // getToolsWithVars will generate Tools from tools config

internal/pkg/configmanager/rawconfig_test.go

@@ -221,15 +221,18 @@ var _ = Describe("rawConfig struct", func() {
 				vars: []byte(`---
 foo1: bar1
 foo2: 123
-foo3: bar3`)}
+foo3: [[foo1]]+[[ foo2]]!
+foo4: [[foo1]]+[[ foo2]]+sep+[[ foo3 ]]!`)}
 		})
 		It("should works fine", func() {
 			varMap, err := r.getVars()
 			Expect(err).NotTo(HaveOccurred())
 			Expect(varMap).NotTo(BeNil())
-			Expect(len(varMap)).To(Equal(3))
+			Expect(len(varMap)).To(Equal(4))
 			Expect(varMap["foo1"]).To(Equal(interface{}("bar1")))
 			Expect(varMap["foo2"]).To(Equal(interface{}(123)))
+			Expect(varMap["foo3"]).To(Equal("bar1+123!"))
+			Expect(varMap["foo4"]).To(Equal("bar1+123+sep+bar1+123!!"))
 		})
 	})
 })

internal/pkg/configmanager/var.go

@@ -0,0 +1,76 @@
+package configmanager
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/devstream-io/devstream/pkg/util/template"
+)
+
+func parseNestedVars(origin map[string]any) (map[string]any, error) {
+	unparsed := make(map[string]any, len(origin))
+	for k, v := range origin {
+		unparsed[k] = v
+	}
+
+	parsed := make(map[string]any, len(origin))
+	updated := true // if any vars were updated in one loop
+
+	// loop until:
+	// 1. all vars have been parsed
+	// 2. or can not render more vars by existing parsed vars
+	for len(unparsed) > 0 && updated {
+		updated = false
+		// use "parsed map" to parse each <key, value> in "unparsed map".
+		// once one value doesn't contain other keys, put it into "parsed map".
+
+		// a util func which implements the "put" steps
+		putParsedValue := func(k string, v any) {
+			parsed[k] = v
+			delete(unparsed, k)
+			updated = true
+		}
+
+		for k, v := range unparsed {
+			// if value is not string or doesn't contain var, just put
+			vString, ok := v.(string)
+			if !ok || !ifContainVar(vString) {
+				putParsedValue(k, v)
+				continue
+			}
+
+			// parse one value with "parsed map"
+			valueParsed, err := template.NewRenderClient(&template.TemplateOption{},
+				template.ContentGetter, template.AddDotForVariablesInConfigProcessor).
+				Render(fmt.Sprintf("%v", v), parsed)
+			// if no error(means this value doesn't import vars in "unparsed map"), put
+			if err == nil {
+				putParsedValue(k, valueParsed)
+			}
+		}
+	}
+
+	// check if vars map is correct
+	if len(unparsed) > 0 {
+		errString := "failed to parse var "
+		var errKeyValues []string
+		for k := range unparsed {
+			errKeyValues = append(errKeyValues, fmt.Sprintf(`<"%s": "%s">`, k, origin[k]))
+		}
+		errString += strings.Join(errKeyValues, ", ")
+		return nil, fmt.Errorf(errString)
+	}
+
+	return parsed, nil
+
+}
+
+// check if value contains other vars
+func ifContainVar(value string) bool {
+	// this function is humble, some case could not be checked probably
+	// e.g. "[[123]]" "[[]]" will be regard as a var
+	if strings.Contains(value, "[[") && strings.Contains(value, "]]") {
+		return true
+	}
+	return false
+}

internal/pkg/configmanager/var_test.go

@@ -0,0 +1,138 @@
+package configmanager
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("parseNestedVars", func() {
+	var (
+		origin, expected, parsed map[string]any
+		err                      error
+	)
+
+	JustBeforeEach(func() {
+		parsed, err = parseNestedVars(origin)
+	})
+
+	When("vars map is correct", func() {
+		When("case is simple(no nested)", func() {
+			BeforeEach(func() {
+				origin = map[string]any{
+					"a": "a",
+					"b": 123,
+				}
+				expected = origin
+			})
+			It("should parse succeed", func() {
+				Expect(err).Should(Succeed())
+				Expect(parsed).Should(Equal(expected))
+			})
+		})
+		When("case is complex(nested once)", func() {
+			BeforeEach(func() {
+				origin = map[string]any{
+					"a": "[[ b]]a",
+					"b": "123",
+				}
+				expected = map[string]any{
+					"a": "123a",
+					"b": "123",
+				}
+			})
+			It("should parse succeed", func() {
+				Expect(err).Should(Succeed())
+				Expect(parsed).Should(Equal(expected))
+			})
+		})
+		When("case is complex(nested many times)", func() {
+			BeforeEach(func() {
+				origin = map[string]any{
+					"a": "[[ b]]a",
+					"b": 123,
+					"c": "[[a]]c",
+					"d": "[[a]]/[[ c ]]/[[b]]",
+				}
+				expected = map[string]any{
+					"a": "123a",
+					"b": 123,
+					"c": "123ac",
+					"d": "123a/123ac/123",
+				}
+			})
+
+			It("should parse succeed", func() {
+				Expect(err).Should(Succeed())
+				Expect(parsed).Should(Equal(expected))
+			})
+		})
+	})
+
+	When("vars map is incorrect", func() {
+		BeforeEach(func() {
+			origin = map[string]any{
+				"a": "[[ b]]a",
+				"b": "123",
+				"c": "[[a]]c[[c]]",
+				"d": "[[a]]/[[ c ]]/[[b]]",
+			}
+			expected = map[string]any{
+				"a": "123a",
+				"b": "123",
+				"c": "123ac",
+				"d": "123a/123ac/123",
+			}
+		})
+
+		It("should return error", func() {
+			Expect(err).Should(HaveOccurred())
+		})
+	})
+})
+
+var _ = Describe("ifContainVar", func() {
+	var (
+		value       string
+		checkResult bool
+	)
+
+	JustBeforeEach(func() {
+		checkResult = ifContainVar(value)
+	})
+
+	When("contains var", func() {
+		AfterEach(func() {
+			Expect(checkResult).To(Equal(true))
+		})
+		Context("case 1", func() {
+			BeforeEach(func() {
+				value = " [[a]]"
+			})
+			It("should check correctly", func() {})
+		})
+		Context("case 2", func() {
+			BeforeEach(func() {
+				value = "[[ b ]]"
+			})
+			It("should check correctly", func() {})
+		})
+	})
+
+	When("doesn't contain var", func() {
+		AfterEach(func() {
+			Expect(checkResult).To(Equal(false))
+		})
+		Context("case 1", func() {
+			BeforeEach(func() {
+				value = " [[a] ]"
+			})
+			It("should check correctly", func() {})
+		})
+		Context("case 2", func() {
+			BeforeEach(func() {
+				value = " [ b ]]"
+			})
+			It("should check correctly", func() {})
+		})
+	})
+})