refactor: use regex for config extract

Signed-off-by: Meng JiaFeng <[email protected]>

Author: steinliber

internal/pkg/configmanager/app.go

@@ -3,9 +3,7 @@ package configmanager
 import (
 	"fmt"
 
-	"gopkg.in/yaml.v3"
-
-	"github.com/devstream-io/devstream/pkg/util/file"
+	"github.com/devstream-io/devstream/pkg/util/log"
 	"github.com/devstream-io/devstream/pkg/util/scm"
 )
 
@@ -27,29 +25,27 @@ type app struct {
 	CDRawConfigs []pipelineRaw `yaml:"cd" mapstructure:"cd"`
 }
 
-func getAppsFromConfigFileWithVarsRendered(fileBytes []byte, vars map[string]any) ([]*app, error) {
-	yamlPath := "$.apps[*]"
-	yamlStrArray, err := file.GetYamlNodeArrayByPath(fileBytes, yamlPath)
+func (a *app) getTools(vars map[string]any, templateMap map[string]string) (Tools, error) {
+	// generate app repo and template repo from scmInfo
+	a.setDefault()
+	repoScaffoldingTool, err := a.getRepoTemplateTool()
 	if err != nil {
-		return nil, err
-	}
-
-	if yamlStrArray == nil {
-		return make([]*app, 0), nil
+		return nil, fmt.Errorf("app[%s] can't get valid repo config: %w", a.Name, err)
 	}
 
-	yamlWithVars, err := renderConfigWithVariables(yamlStrArray.StrOrigin, vars)
+	// get ci/cd pipelineTemplates
+	appVars := a.Spec.merge(vars)
+	tools, err := a.generateCICDToolsFromAppConfig(templateMap, appVars)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("app[%s] get pipeline tools failed: %w", a.Name, err)
 	}
-
-	var retTApps = make([]*app, 0)
-	err = yaml.Unmarshal(yamlWithVars, &retTApps)
-	if err != nil {
-		return nil, err
+	if repoScaffoldingTool != nil {
+		tools = append(tools, repoScaffoldingTool)
 	}
 
-	return retTApps, nil
+	log.Debugf("Have got %d tools from app %s.", len(tools), a.Name)
+
+	return tools, nil
 }
 
 // getAppPipelineTool generate ci/cd tools from app config

internal/pkg/configmanager/app_test.go

@@ -1,27 +1,56 @@
 package configmanager
 
 import (
+	"fmt"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+
+	"github.com/devstream-io/devstream/pkg/util/scm"
 )
 
-var _ = Describe("getToolsFromConfigFileWithVarsRendered", func() {
-	const appsConfig = `---
-apps:
-- name: app-1
-  cd:
-  - type: template
-    vars:
-      app: [[ appName ]]
-`
-	When("get apps from config file", func() {
-		It("should return config with vars", func() {
-			apps, err := getAppsFromConfigFileWithVarsRendered([]byte(appsConfig), map[string]any{"appName": interface{}("app-1")})
-			Expect(err).NotTo(HaveOccurred())
-			Expect(apps).NotTo(BeNil())
-			Expect(len(apps)).To(Equal(1))
-			Expect(len(apps[0].CDRawConfigs)).To(Equal(1))
-			Expect(apps[0].CDRawConfigs[0].Vars["app"]).To(Equal(interface{}("app-1")))
+var _ = Describe("app struct", func() {
+	var (
+		a           *app
+		appName     string
+		vars        map[string]any
+		templateMap map[string]string
+	)
+	BeforeEach(func() {
+		appName = "test_app"
+		vars = map[string]any{}
+		templateMap = map[string]string{}
+	})
+	Context("getTools method", func() {
+		When("repo is not valid", func() {
+			BeforeEach(func() {
+				a = &app{Name: appName}
+			})
+			It("should return error", func() {
+				_, err := a.getTools(vars, templateMap)
+				Expect(err).Should(HaveOccurred())
+				Expect(err.Error()).Should(ContainSubstring(fmt.Sprintf("app[%s] can't get valid repo config", appName)))
+			})
+		})
+		When("ci/cd template is not valid", func() {
+			BeforeEach(func() {
+				a = &app{
+					Repo: &scm.SCMInfo{
+						CloneURL: "http://test.com/test/test_app",
+					},
+					CIRawConfigs: []pipelineRaw{
+						{
+							Type:         "template",
+							TemplateName: "not_exist",
+						},
+					},
+				}
+			})
+			It("should return error", func() {
+				_, err := a.getTools(vars, templateMap)
+				Expect(err).Should(HaveOccurred())
+				Expect(err.Error()).Should(ContainSubstring("not found in pipelineTemplates"))
+			})
 		})
 	})
 })

internal/pkg/configmanager/config.go

@@ -4,65 +4,13 @@ import (
 	"fmt"
 
 	"gopkg.in/yaml.v3"
-
-	"github.com/devstream-io/devstream/pkg/util/log"
 )
 
 // Config is a general config in DevStream.
 type Config struct {
 	Config CoreConfig     `yaml:"config"`
 	Vars   map[string]any `yaml:"vars"`
 	Tools  Tools          `yaml:"tools"`
-	Apps   []*app         `yaml:"apps"`
-	// We'll read the pipeline templates from config file and render it to pipelineTemplateMap in no time.
-	//PipelineTemplates   []*pipelineTemplate `yaml:"-"`
-	pipelineTemplateMap map[string]string `yaml:"-"`
-}
-
-func (c *Config) getToolsFromApps() (Tools, error) {
-	var tools Tools
-	for _, a := range c.Apps {
-		appTools, err := c.getToolsFromApp(a)
-		if err != nil {
-			return nil, err
-		}
-		tools = append(tools, appTools...)
-	}
-	return tools, nil
-}
-
-func (c *Config) getToolsFromApp(a *app) (Tools, error) {
-	// generate app repo and template repo from scmInfo
-	a.setDefault()
-	repoScaffoldingTool, err := a.getRepoTemplateTool()
-	if err != nil {
-		return nil, fmt.Errorf("app[%s] get repo failed: %w", a.Name, err)
-	}
-
-	// get ci/cd pipelineTemplates
-	appVars := a.Spec.merge(c.Vars)
-	tools, err := a.generateCICDToolsFromAppConfig(c.pipelineTemplateMap, appVars)
-	if err != nil {
-		return nil, fmt.Errorf("app[%s] get pipeline tools failed: %w", a.Name, err)
-	}
-	if repoScaffoldingTool != nil {
-		tools = append(tools, repoScaffoldingTool)
-	}
-
-	// all tools from apps should depend on the original tools,
-	// because dtm will execute all original tools first, then execute all tools from apps
-	for _, toolFromApps := range tools {
-		for _, t := range c.Tools {
-			toolFromApps.DependsOn = append(toolFromApps.DependsOn, t.KeyWithNameAndInstanceID())
-		}
-	}
-
-	log.Debugf("Have got %d tools from app %s.", len(tools), a.Name)
-	for i, t := range tools {
-		log.Debugf("Tool %d: %v", i+1, t)
-	}
-
-	return tools, nil
 }
 
 func (c *Config) renderInstanceIDtoOptions() {

internal/pkg/configmanager/configmanager.go

@@ -20,25 +20,17 @@ func NewManager(configFilePath string) *Manager {
 // LoadConfig is the only method that the caller of Manager needs to be concerned with, and this method returns a *Config finally.
 // The main workflow of this method is:
 // 1. Get the original config from the config file specified by ConfigFilePath;
-// 2. Parsing tools from the apps, and merge that tools to Config.Tools;
-// 3. Validation.
+// 2. Validation.
 func (m *Manager) LoadConfig() (*Config, error) {
-	// step 1
+	// step 1: get config
 	c, err := m.getConfigFromFileWithGlobalVars()
 	if err != nil {
 		return nil, err
 	}
-
-	// step 2
-	appTools, err := c.getToolsFromApps()
-	if err != nil {
-		return nil, err
-	}
-
-	c.Tools = append(c.Tools, appTools...)
+	// set instanceID in options
 	c.renderInstanceIDtoOptions()
 
-	// step 3
+	// step 2: check config is valid
 	if err = c.validate(); err != nil {
 		return nil, err
 	}
@@ -50,50 +42,50 @@ func (m *Manager) LoadConfig() (*Config, error) {
 // 1. render the global variables to Config.Tools and Config.Apps
 // 2. transfer the PipelineTemplates to Config.pipelineTemplateMap, it's map[string]string type.
 // We can't render the original config file to Config.PipelineTemplates directly for the:
-//   1. variables rendered must be before the yaml.Unmarshal() called for the [[ foo ]] will be treated as a two-dimensional array by the yaml parser;
-//   2. the variables used([[ foo ]]) in the Config.PipelineTemplates can be defined in the Config.Apps or Config.Vars;
-//   3. pipeline templates are used in apps, so it would be more appropriate to refer to pipeline templates when dealing with apps
+//  1. variables rendered must be before the yaml.Unmarshal() called for the [[ foo ]] will be treated as a two-dimensional array by the yaml parser;
+//  2. the variables used([[ foo ]]) in the Config.PipelineTemplates can be defined in the Config.Apps or Config.Vars;
 func (m *Manager) getConfigFromFileWithGlobalVars() (*Config, error) {
 	configBytes, err := os.ReadFile(m.ConfigFilePath)
 	if err != nil {
 		return nil, err
 	}
 
-	// global variables
-	vars, err := getVarsFromConfigFile(configBytes)
+	// extract top raw config struct from config text
+	r, err := newRawConfigFromConfigBytes(configBytes)
+	if err != nil {
+		return nil, err
+	}
+	// 1. get global variables
+	vars, err := r.getVars()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get variables from config file. Error: %w", err)
 	}
 
-	// tools with global variables rendered
-	tools, err := getToolsFromConfigFileWithVarsRendered(configBytes, vars)
+	// 2. tools with global variables rendered
+	tools, err := r.getToolsWithVars(vars)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get tools from config file. Error: %w", err)
 	}
 
-	// apps with global variables rendered
-	apps, err := getAppsFromConfigFileWithVarsRendered(configBytes, vars)
+	// 3. apps tools with global variables rendered
+	appTools, err := r.getAppToolsWithVars(vars)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get apps from config file. Error: %w", err)
 	}
+	// all tools from apps should depend on the original tools,
+	// because dtm will execute all original tools first, then execute all tools from apps
+	appTools.updateToolDepends(tools)
+	tools = append(tools, appTools...)
 
-	// pipelineTemplateMap transfer from PipelineTemplates
-	pipelineTemplateMap, err := getPipelineTemplatesMapFromConfigFile(configBytes)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get pipelineTemplatesMap from config file. Error: %w", err)
-	}
-
-	// coreConfig without any changes
-	coreConfig, err := getCoreConfigFromConfigFile(configBytes)
+	// 4. coreConfig without any changes
+	coreConfig, err := r.getConfig()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get coreConfig from config file. Error: %w", err)
 	}
 
 	return &Config{
-		Config:              *coreConfig,
-		Vars:                vars,
-		Tools:               tools,
-		Apps:                apps,
-		pipelineTemplateMap: pipelineTemplateMap,
+		Config: *coreConfig,
+		Vars:   vars,
+		Tools:  tools,
 	}, nil
 }

internal/pkg/configmanager/configmanager_test.go

@@ -218,7 +218,7 @@ var _ = Describe("LoadConfig", func() {
 	})
 
 	When("load a config file", func() {
-		It("should return 5 tools", func() {
+		It("should return  tools", func() {
 			mgr := NewManager(filepath.Join(tmpWorkDir, "config.yaml"))
 			cfg, err := mgr.LoadConfig()
 			Expect(err).NotTo(HaveOccurred())
@@ -274,9 +274,7 @@ var _ = Describe("getConfigFromFileWithGlobalVars", func() {
 			Expect(err).NotTo(HaveOccurred())
 			Expect(cfg.Config.State.Backend).To(Equal("local"))
 			Expect(cfg.Vars["foo1"]).To(Equal("bar1"))
-			Expect(len(cfg.Apps)).To(Equal(1))
-			Expect(cfg.Apps[0].Name).To(Equal("service-a"))
-			Expect(len(cfg.Tools)).To(Equal(2))
+			Expect(len(cfg.Tools)).To(Equal(5))
 			Expect(cfg.Tools[1].Name).To(Equal("plugin2"))
 		})
 	})

internal/pkg/configmanager/coreconfig.go

@@ -1,11 +1,5 @@
 package configmanager
 
-import (
-	"gopkg.in/yaml.v3"
-
-	"github.com/devstream-io/devstream/pkg/util/file"
-)
-
 type CoreConfig struct {
 	State *State `yaml:"state"`
 }
@@ -29,18 +23,3 @@ type StateConfigOptions struct {
 	Namespace string `yaml:"namespace"`
 	ConfigMap string `yaml:"configmap"`
 }
-
-func getCoreConfigFromConfigFile(fileBytes []byte) (*CoreConfig, error) {
-	yamlPath := "$.config"
-	yamlStr, err := file.GetYamlNodeStrByPath(fileBytes, yamlPath)
-	if err != nil {
-		return nil, err
-	}
-
-	var config *CoreConfig
-	err = yaml.Unmarshal([]byte(yamlStr), &config)
-	if err != nil {
-		return nil, err
-	}
-	return config, nil
-}