fix: numeric type variables are parsed as strings

Signed-off-by: Daniel Hu <[email protected]>

Author: daniel-hutao

internal/pkg/configmanager/app.go

@@ -3,6 +3,9 @@ package configmanager
 import (
 	"fmt"
 
+	"gopkg.in/yaml.v3"
+
+	"github.com/devstream-io/devstream/pkg/util/file"
 	"github.com/devstream-io/devstream/pkg/util/scm"
 )
 
@@ -12,7 +15,7 @@ const (
 
 type repoTemplate struct {
 	*scm.SCMInfo `yaml:",inline"`
-	Vars         map[string]any `yaml:"vars"`
+	Vars         RawOptions `yaml:"vars"`
 }
 
 type app struct {
@@ -24,6 +27,31 @@ type app struct {
 	CDRawConfigs []pipelineRaw `yaml:"cd" mapstructure:"cd"`
 }
 
+func getAppsFromConfigFileWithVarsRendered(fileBytes []byte, vars map[string]any) ([]*app, error) {
+	yamlPath := "$.apps[*]"
+	yamlStrArray, err := file.GetYamlNodeArrayByPath(fileBytes, yamlPath)
+	if err != nil {
+		return nil, err
+	}
+
+	if yamlStrArray == nil {
+		return make([]*app, 0), nil
+	}
+
+	yamlWithVars, err := renderConfigWithVariables(yamlStrArray.StrOrigin, vars)
+	if err != nil {
+		return nil, err
+	}
+
+	var retTApps = make([]*app, 0)
+	err = yaml.Unmarshal(yamlWithVars, &retTApps)
+	if err != nil {
+		return nil, err
+	}
+
+	return retTApps, nil
+}
+
 // getAppPipelineTool generate ci/cd tools from app config
 func (a *app) generateCICDToolsFromAppConfig(templateMap map[string]string, appVars map[string]any) (Tools, error) {
 	allPipelineRaw := append(a.CIRawConfigs, a.CDRawConfigs...)
@@ -57,11 +85,14 @@ func (a *app) getRepoTemplateTool() (*Tool, error) {
 		if err != nil {
 			return nil, fmt.Errorf("configmanager[app] parse repoTemplate failed: %w", err)
 		}
+		if a.RepoTemplate.Vars == nil {
+			a.RepoTemplate.Vars = make(RawOptions)
+		}
 		return newTool(
 			repoScaffoldingPluginName, a.Name, RawOptions{
 				"destinationRepo": RawOptions(appRepo.Encode()),
 				"sourceRepo":      RawOptions(templateRepo.Encode()),
-				"vars":            RawOptions(a.RepoTemplate.Vars),
+				"vars":            a.RepoTemplate.Vars,
 			},
 		), nil
 	}

internal/pkg/configmanager/app_test.go

@@ -0,0 +1,27 @@
+package configmanager
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("getToolsFromConfigFileWithVarsRendered", func() {
+	const appsConfig = `---
+apps:
+- name: app-1
+  cd:
+  - type: template
+    vars:
+      app: [[ appName ]]
+`
+	When("get apps from config file", func() {
+		It("should return config with vars", func() {
+			apps, err := getAppsFromConfigFileWithVarsRendered([]byte(appsConfig), map[string]any{"appName": interface{}("app-1")})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(apps).NotTo(BeNil())
+			Expect(len(apps)).To(Equal(1))
+			Expect(len(apps[0].CDRawConfigs)).To(Equal(1))
+			Expect(apps[0].CDRawConfigs[0].Vars["app"]).To(Equal(interface{}("app-1")))
+		})
+	})
+})

internal/pkg/configmanager/config.go

@@ -10,23 +10,19 @@ import (
 
 // Config is a general config in DevStream.
 type Config struct {
-	Config              CoreConfig         `yaml:"config"`
-	Vars                map[string]any     `yaml:"vars"`
-	Tools               Tools              `yaml:"tools"`
-	Apps                []app              `yaml:"apps"`
-	PipelineTemplates   []pipelineTemplate `yaml:"pipelineTemplates"`
-	pipelineTemplateMap map[string]string  `yaml:"-"`
+	Config CoreConfig     `yaml:"config"`
+	Vars   map[string]any `yaml:"vars"`
+	Tools  Tools          `yaml:"tools"`
+	Apps   []*app         `yaml:"apps"`
+	// We'll read the pipeline templates from config file and render it to pipelineTemplateMap in no time.
+	//PipelineTemplates   []*pipelineTemplate `yaml:"-"`
+	pipelineTemplateMap map[string]string `yaml:"-"`
 }
 
 func (c *Config) getToolsFromApps() (Tools, error) {
-	err := c.renderPipelineTemplateMap()
-	if err != nil {
-		return nil, err
-	}
-
 	var tools Tools
 	for _, a := range c.Apps {
-		appTools, err := c.getToolsWithVarsFromApp(a)
+		appTools, err := c.getToolsFromApp(a)
 		if err != nil {
 			return nil, err
 		}
@@ -35,92 +31,45 @@ func (c *Config) getToolsFromApps() (Tools, error) {
 	return tools, nil
 }
 
-func (c *Config) getToolsWithVarsFromApp(a app) (Tools, error) {
-	appStr, err := yaml.Marshal(a)
+func (c *Config) getToolsFromApp(a *app) (Tools, error) {
+	// generate app repo and template repo from scmInfo
+	a.setDefault()
+	repoScaffoldingTool, err := a.getRepoTemplateTool()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("app[%s] get repo failed: %w", a.Name, err)
 	}
 
-	// 1. render appStr with Vars
-	appRenderStr, err := renderConfigWithVariables(string(appStr), c.Vars)
+	// get ci/cd pipelineTemplates
+	appVars := a.Spec.merge(c.Vars)
+	tools, err := a.generateCICDToolsFromAppConfig(c.pipelineTemplateMap, appVars)
 	if err != nil {
-		log.Debugf("configmanager/app %s render globalVars %+v failed", appRenderStr, c.Vars)
-		return nil, fmt.Errorf("app render globalVars failed: %w", err)
-	}
-
-	// 2. unmarshal app config for render pipelineTemplate
-	var rawApp app
-	if err = yaml.Unmarshal(appRenderStr, &rawApp); err != nil {
-		return nil, fmt.Errorf("app parse yaml failed: %w", err)
-	}
-
-	// 3. generate app repo and template repo from scmInfo
-	rawApp.setDefault()
-	repoScaffoldingTool, err := rawApp.getRepoTemplateTool()
-	if err != nil {
-		return nil, fmt.Errorf("app[%s] get repo failed: %w", rawApp.Name, err)
-	}
-
-	// 4. get ci/cd pipelineTemplates
-	appVars := rawApp.Spec.merge(c.Vars)
-	tools, err := rawApp.generateCICDToolsFromAppConfig(c.pipelineTemplateMap, appVars)
-	if err != nil {
-		return nil, fmt.Errorf("app[%s] get pipeline tools failed: %w", rawApp.Name, err)
+		return nil, fmt.Errorf("app[%s] get pipeline tools failed: %w", a.Name, err)
 	}
 	if repoScaffoldingTool != nil {
 		tools = append(tools, repoScaffoldingTool)
 	}
 
-	// 5. all tools from apps should depend on the original tools,
-	//    because dtm will execute all original tools first, then execute all tools from apps
+	// all tools from apps should depend on the original tools,
+	// because dtm will execute all original tools first, then execute all tools from apps
 	for _, toolFromApps := range tools {
 		for _, t := range c.Tools {
 			toolFromApps.DependsOn = append(toolFromApps.DependsOn, t.KeyWithNameAndInstanceID())
 		}
 	}
 
-	log.Debugf("Have got %d tools from app %s.", len(tools), rawApp.Name)
+	log.Debugf("Have got %d tools from app %s.", len(tools), a.Name)
 	for i, t := range tools {
 		log.Debugf("Tool %d: %v", i+1, t)
 	}
 
 	return tools, nil
 }
 
-func (c *Config) renderToolsWithVars() error {
-	toolsStr, err := yaml.Marshal(c.Tools)
-	if err != nil {
-		return err
-	}
-
-	toolsStrWithVars, err := renderConfigWithVariables(string(toolsStr), c.Vars)
-	if err != nil {
-		return err
-	}
-
-	var tools Tools
-	if err = yaml.Unmarshal(toolsStrWithVars, &tools); err != nil {
-		return err
-	}
-	c.Tools = tools
-
-	return nil
-}
-
-func (c *Config) renderPipelineTemplateMap() error {
-	c.pipelineTemplateMap = make(map[string]string)
-	for _, pt := range c.PipelineTemplates {
-		tplBytes, err := yaml.Marshal(pt)
-		if err != nil {
-			return err
-		}
-		c.pipelineTemplateMap[pt.Name] = string(tplBytes)
-	}
-	return nil
-}
-
 func (c *Config) renderInstanceIDtoOptions() {
 	for _, t := range c.Tools {
+		if t.Options == nil {
+			t.Options = make(RawOptions)
+		}
 		t.Options["instanceID"] = t.InstanceID
 	}
 }

internal/pkg/configmanager/configmanager.go

@@ -1,12 +1,8 @@
 package configmanager
 
 import (
+	"fmt"
 	"os"
-	"regexp"
-
-	"gopkg.in/yaml.v3"
-
-	"github.com/devstream-io/devstream/pkg/util/log"
 )
 
 // Manager is used to load the config file from the ConfigFilePath and finally get the Config object.
@@ -28,7 +24,7 @@ func NewManager(configFilePath string) *Manager {
 // 3. Validation.
 func (m *Manager) LoadConfig() (*Config, error) {
 	// step 1
-	c, err := m.getConfigFromFile()
+	c, err := m.getConfigFromFileWithGlobalVars()
 	if err != nil {
 		return nil, err
 	}
@@ -39,10 +35,6 @@ func (m *Manager) LoadConfig() (*Config, error) {
 		return nil, err
 	}
 
-	err = c.renderToolsWithVars()
-	if err != nil {
-		return nil, err
-	}
 	c.Tools = append(c.Tools, appTools...)
 	c.renderInstanceIDtoOptions()
 
@@ -54,26 +46,54 @@ func (m *Manager) LoadConfig() (*Config, error) {
 	return c, nil
 }
 
-// getConfigFromFile gets Config from the config file specified by Manager.ConfigFilePath
-func (m *Manager) getConfigFromFile() (*Config, error) {
+// getConfigFromFileWithGlobalVars gets Config from the config file specified by Manager.ConfigFilePath, then:
+// 1. render the global variables to Config.Tools and Config.Apps
+// 2. transfer the PipelineTemplates to Config.pipelineTemplateMap, it's map[string]string type.
+// We can't render the original config file to Config.PipelineTemplates directly for the:
+//   1. variables rendered must be before the yaml.Unmarshal() called for the [[ foo ]] will be treated as a two-dimensional array by the yaml parser;
+//   2. the variables used([[ foo ]]) in the Config.PipelineTemplates can be defined in the Config.Apps or Config.Vars;
+//   3. pipeline templates are used in apps, so it would be more appropriate to refer to pipeline templates when dealing with apps
+func (m *Manager) getConfigFromFileWithGlobalVars() (*Config, error) {
 	configBytes, err := os.ReadFile(m.ConfigFilePath)
 	if err != nil {
 		return nil, err
 	}
 
-	configBytesEscaped := escapeBrackets(configBytes)
+	// global variables
+	vars, err := getVarsFromConfigFile(configBytes)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get variables from config file. Error: %w", err)
+	}
 
-	var c Config
-	if err = yaml.Unmarshal(configBytesEscaped, &c); err != nil {
-		log.Errorf("Please verify the format of your config. Error: %s.", err)
-		return nil, err
+	// tools with global variables rendered
+	tools, err := getToolsFromConfigFileWithVarsRendered(configBytes, vars)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get tools from config file. Error: %w", err)
 	}
 
-	return &c, nil
-}
+	// apps with global variables rendered
+	apps, err := getAppsFromConfigFileWithVarsRendered(configBytes, vars)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get apps from config file. Error: %w", err)
+	}
+
+	// pipelineTemplateMap transfer from PipelineTemplates
+	pipelineTemplateMap, err := getPipelineTemplatesMapFromConfigFile(configBytes)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get pipelineTemplatesMap from config file. Error: %w", err)
+	}
+
+	// coreConfig without any changes
+	coreConfig, err := getCoreConfigFromConfigFile(configBytes)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get coreConfig from config file. Error: %w", err)
+	}
 
-// escapeBrackets is used to escape []byte(": [[xxx]]xxx\n") to []byte(": \"[[xxx]]\"xxx\n")
-func escapeBrackets(param []byte) []byte {
-	re := regexp.MustCompile(`([^:]+:)(\s*)((\[\[[^\]]+\]\][^\s\[]*)+)[^\s#\n]*`)
-	return re.ReplaceAll(param, []byte("$1$2\"$3\""))
+	return &Config{
+		Config:              *coreConfig,
+		Vars:                vars,
+		Tools:               tools,
+		Apps:                apps,
+		pipelineTemplateMap: pipelineTemplateMap,
+	}, nil
 }