feat: config support directory

aFlyBird0 · aFlyBird0 · commit 73025d4f7db7 · 2022-12-09T15:03:02.000+08:00
Signed-off-by: Bird &lt;aflybird0@gmail.com&gt;
diff --git a/cmd/devstream/apply.go b/cmd/devstream/apply.go
@@ -33,7 +33,7 @@ func applyCMDFunc(cmd *cobra.Command, args []string) {
 }
 
 func init() {
-	applyCMD.Flags().StringVarP(&configFilePath, configFlagName, "f", "config.yaml", "config file")
+	applyCMD.Flags().StringVarP(&configFilePath, configFlagName, "f", "", "config file or directory")
 	applyCMD.Flags().StringVarP(&pluginDir, pluginDirFlagName, "d", defaultPluginDir, "plugins directory")
 	applyCMD.Flags().BoolVarP(&continueDirectly, "yes", "y", false, "apply directly without confirmation")
 
diff --git a/cmd/devstream/delete.go b/cmd/devstream/delete.go
@@ -32,7 +32,7 @@ func deleteCMDFunc(cmd *cobra.Command, args []string) {
 
 func init() {
 	deleteCMD.Flags().BoolVarP(&isForceDelete, "force", "", false, "force delete by config")
-	deleteCMD.Flags().StringVarP(&configFilePath, configFlagName, "f", "config.yaml", "config file")
+	deleteCMD.Flags().StringVarP(&configFilePath, configFlagName, "f", "", "config file or directory")
 	deleteCMD.Flags().StringVarP(&pluginDir, pluginDirFlagName, "d", defaultPluginDir, "plugins directory")
 	deleteCMD.Flags().BoolVarP(&continueDirectly, "yes", "y", false, "delete directly without confirmation")
 
diff --git a/cmd/devstream/destroy.go b/cmd/devstream/destroy.go
@@ -30,7 +30,7 @@ func destroyCMDFunc(cmd *cobra.Command, args []string) {
 
 func init() {
 	destroyCMD.Flags().BoolVarP(&isForceDestroy, "force", "", false, "force destroy by config")
-	destroyCMD.Flags().StringVarP(&configFilePath, configFlagName, "f", "config.yaml", "config file")
+	destroyCMD.Flags().StringVarP(&configFilePath, configFlagName, "f", "", "config file or directory")
 	destroyCMD.Flags().StringVarP(&pluginDir, pluginDirFlagName, "d", defaultPluginDir, "plugins directory")
 	destroyCMD.Flags().BoolVarP(&continueDirectly, "yes", "y", false, "destroy directly without confirmation")
 
diff --git a/cmd/devstream/init.go b/cmd/devstream/init.go
@@ -118,7 +118,7 @@ func GetPluginsFromFlags() (tools configmanager.Tools, err error) {
 
 func init() {
 	// flags for init from config file
-	initCMD.Flags().StringVarP(&configFilePath, configFlagName, "f", "config.yaml", "config file")
+	initCMD.Flags().StringVarP(&configFilePath, configFlagName, "f", "", "config file or directory")
 	initCMD.Flags().StringVarP(&pluginDir, pluginDirFlagName, "d", defaultPluginDir, "plugins directory")
 
 	// downloading specific plugins from flags
diff --git a/internal/pkg/configmanager/configmanager.go b/internal/pkg/configmanager/configmanager.go
@@ -2,19 +2,20 @@ package configmanager
 
 import (
 	"fmt"
-	"os"
 	"strings"
+
+	"github.com/devstream-io/devstream/pkg/util/file"
 )
 
 // Manager is used to load the config file from the ConfigFilePath and finally get the Config object.
 type Manager struct {
 	ConfigFilePath string
 }
 
-// NewManager takes configFilePath, then return a *Manager object.
-func NewManager(configFilePath string) *Manager {
+// NewManager takes configFilePath(file or directory), then return a *Manager object.
+func NewManager(configPath string) *Manager {
 	return &Manager{
-		ConfigFilePath: configFilePath,
+		ConfigFilePath: configPath,
 	}
 }
 
@@ -46,7 +47,7 @@ func (m *Manager) LoadConfig() (*Config, error) {
 //  1. variables rendered must be before the yaml.Unmarshal() called for the [[ foo ]] will be treated as a two-dimensional array by the yaml parser;
 //  2. the variables used([[ foo ]]) in the Config.PipelineTemplates can be defined in the Config.Apps or Config.Vars;
 func (m *Manager) getConfigFromFileWithGlobalVars() (*Config, error) {
-	configBytes, err := os.ReadFile(m.ConfigFilePath)
+	configBytes, err := file.ReadYamls(m.ConfigFilePath)
 	if err != nil {
 		return nil, err
 	}
diff --git a/internal/pkg/plugin/installer/ci/cifile/server/githubci.go b/internal/pkg/plugin/installer/ci/cifile/server/githubci.go
@@ -22,7 +22,7 @@ func (g *GitHubWorkflow) CIFilePath() string {
 	return filepath.Join(ciGitHubWorkConfigLocation, ciGithubTempName)
 }
 
-func (g *GitHubWorkflow) FilterCIFilesFunc() file.DirFIleFilterFunc {
+func (g *GitHubWorkflow) FilterCIFilesFunc() file.DirFileFilterFunc {
 	return func(filePath string, isDir bool) bool {
 		// not process dir
 		if isDir {
diff --git a/internal/pkg/plugin/installer/ci/cifile/server/gitlabci.go b/internal/pkg/plugin/installer/ci/cifile/server/gitlabci.go
@@ -19,7 +19,7 @@ func (g *GitLabCI) CIFilePath() string {
 	return ciGitLabConfigLocation
 }
 
-func (g *GitLabCI) FilterCIFilesFunc() file.DirFIleFilterFunc {
+func (g *GitLabCI) FilterCIFilesFunc() file.DirFileFilterFunc {
 	return func(filePath string, isDir bool) bool {
 		// not process dir
 		if isDir {
diff --git a/internal/pkg/plugin/installer/ci/cifile/server/jenkinsci.go b/internal/pkg/plugin/installer/ci/cifile/server/jenkinsci.go
@@ -20,7 +20,7 @@ func (j *JenkinsPipeline) CIFilePath() string {
 }
 
 // FilterCIFilesFunc only get file with name Jenkinsfile
-func (j *JenkinsPipeline) FilterCIFilesFunc() file.DirFIleFilterFunc {
+func (j *JenkinsPipeline) FilterCIFilesFunc() file.DirFileFilterFunc {
 	return func(filePath string, isDir bool) bool {
 		// not process dir
 		if isDir {
diff --git a/internal/pkg/plugin/installer/ci/cifile/server/server.go b/internal/pkg/plugin/installer/ci/cifile/server/server.go
@@ -15,7 +15,7 @@ type CIServerOptions interface {
 	CIFilePath() string
 
 	// FilterCIFilesFunc returns a filter function to select ci config file
-	FilterCIFilesFunc() file.DirFIleFilterFunc
+	FilterCIFilesFunc() file.DirFileFilterFunc
 	// GetGitNameFunc returns a function to transform file path to git name of ci config file
 	GetGitNameFunc() file.DirFileNameFunc
 }
diff --git a/pkg/util/file/dir.go b/pkg/util/file/dir.go
@@ -8,9 +8,9 @@ import (
 	"github.com/devstream-io/devstream/pkg/util/log"
 )
 
-// DirFIleFilterFunc is used to filter files when walk directory
+// DirFileFilterFunc is used to filter files when walk directory
 // if this func return false, this file's content will not return
-type DirFIleFilterFunc func(filePath string, isDir bool) bool
+type DirFileFilterFunc func(filePath string, isDir bool) bool
 
 // DirFileContentFunc is used to get file content then return this content
 type DirFileContentFunc func(filePath string) ([]byte, error)
@@ -21,7 +21,7 @@ type DirFileNameFunc func(filePath, srcPath string) string
 
 // GetFileMapByWalkDir will walk in directory return contentMap
 func GetFileMapByWalkDir(
-	srcPath string, filterFunc DirFIleFilterFunc, fileNameFunc DirFileNameFunc, processFunc DirFileContentFunc,
+	srcPath string, filterFunc DirFileFilterFunc, fileNameFunc DirFileNameFunc, processFunc DirFileContentFunc,
 ) (map[string][]byte, error) {
 	contentMap := make(map[string][]byte)
 	if err := filepath.Walk(srcPath, func(path string, info fs.FileInfo, err error) error {
@@ -54,7 +54,7 @@ func GetFileMapByWalkDir(
 // if srcPath is a directory, it will invoke GetFileMapByWalkDir to get content map
 // if srcPath is a file, it will use fileNameFunc and fileContentFunc to create a map
 func GetFileMap(
-	srcPath string, filterFunc DirFIleFilterFunc, fileNameFunc DirFileNameFunc, fileContentFunc DirFileContentFunc,
+	srcPath string, filterFunc DirFileFilterFunc, fileNameFunc DirFileNameFunc, fileContentFunc DirFileContentFunc,
 ) (map[string][]byte, error) {
 	pathInfo, err := os.Stat(srcPath)
 	if err != nil {
diff --git a/pkg/util/file/dir_test.go b/pkg/util/file/dir_test.go
@@ -14,8 +14,8 @@ var _ = Describe("GetFileMapByWalkDir func", func() {
 	var (
 		tempDir, exepectKey    string
 		testContent            []byte
-		mockFilterSuccessFunc  file.DirFIleFilterFunc
-		mockFilterFailedFunc   file.DirFIleFilterFunc
+		mockFilterSuccessFunc  file.DirFileFilterFunc
+		mockFilterFailedFunc   file.DirFileFilterFunc
 		mockGetFileNameFunc    file.DirFileNameFunc
 		mockProcessFailedFunc  file.DirFileContentFunc
 		mockProcessSuccessFunc file.DirFileContentFunc
diff --git a/pkg/util/file/yaml.go b/pkg/util/file/yaml.go
@@ -3,6 +3,10 @@ package file
 import (
 	"bytes"
 	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
 
 	yamlUtil "github.com/goccy/go-yaml"
 	"github.com/goccy/go-yaml/ast"
@@ -84,3 +88,40 @@ func getYamlAstNode(content []byte, path string) (ast.Node, error) {
 	}
 	return node, nil
 }
+
+// ReadYamls reads file or files from dir whose suffix is yaml or yml
+// and returns the content of the files without "---" separator
+func ReadYamls(path string) ([]byte, error) {
+	stat, err := os.Stat(path)
+	if err != nil {
+		return nil, err
+	}
+	var contents []byte
+	if stat.IsDir() {
+		filterYaml := func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			if d.IsDir() {
+				return nil
+			}
+			if filepath.Ext(path) == ".yaml" || filepath.Ext(path) == ".yml" {
+				content, err := os.ReadFile(path)
+				if err != nil {
+					return err
+				}
+				contents = append(contents, content...)
+			}
+			return nil
+		}
+		err = filepath.WalkDir(path, filterYaml)
+	} else {
+		contents, err = os.ReadFile(path)
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	return []byte(strings.ReplaceAll(string(contents), "\n---\n", "\n")), nil
+}
diff --git a/pkg/util/file/yamlReader_test.go b/pkg/util/file/yamlReader_test.go
@@ -0,0 +1,27 @@
+package file
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+)
+
+func TestYamlReaderFile(t *testing.T) {
+	content, err := ReadYamls("config-gitops.yaml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(string(content))
+}
+
+func TestYamlReaderDir(t *testing.T) {
+	content, err := ReadYamls("tmp")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(string(content))
+}
+
+var _ = Describe("ReadYamls func", func() {
+
+})
diff --git a/pkg/util/file/yaml_test.go b/pkg/util/file/yaml_test.go
@@ -1,6 +1,12 @@
 package file_test
 
 import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	mapset "github.com/deckarep/golang-set/v2"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
@@ -202,3 +208,75 @@ tests:
 		Expect(node).Should(Equal("  name: plugin1\n  instanceID: default\n  options:\n    key1: [[var]]"))
 	})
 })
+
+var _ = Describe("ReadYamls func", func() {
+	var contents = []string{
+		"test_content1\n---\ntest_content1-1\n",
+		"test_content2\n",
+		"test_content3\n",
+		"test_content4\n---\n",
+	}
+	var contentsWithoutSeprator = []string{
+		"test_content1",
+		"test_content1-1",
+		"test_content2",
+		"test_content3",
+		"test_content4",
+	}
+
+	var (
+		tempDir, filePath string
+	)
+
+	JustAfterEach(func() {
+		dataReadBytes, err := file.ReadYamls(filePath)
+		Expect(err).Error().ShouldNot(HaveOccurred())
+		dataReadStrSlice := strings.Split(strings.TrimSpace(string(dataReadBytes)), "\n")
+		Expect(mapset.NewSet(dataReadStrSlice...).
+			Equal(mapset.NewSet(contentsWithoutSeprator...))).
+			Should(BeTrue(),
+				fmt.Sprintf("dataRead: %v\noriginContent: %v", dataReadStrSlice, contentsWithoutSeprator))
+	})
+
+	Context("read from file", func() {
+		BeforeEach(func() {
+			tempDir = GinkgoT().TempDir()
+			filePath = filepath.Join(tempDir, "test.yaml")
+			f, err := os.Create(filePath)
+			Expect(err).Error().ShouldNot(HaveOccurred())
+			defer f.Close()
+			for _, content := range contents {
+				_, err := f.WriteString(content)
+				Expect(err).Error().ShouldNot(HaveOccurred())
+			}
+		})
+		It("should return all contents", func() {
+			filePath = filepath.Join(tempDir, "test.yaml")
+		})
+	})
+
+	Context("read from dir", func() {
+		BeforeEach(func() {
+			tempDir = GinkgoT().TempDir()
+			for i, content := range contents[:len(contents)-1] {
+				filePath = filepath.Join(tempDir, fmt.Sprintf("test-%d.yaml", i))
+				f, err := os.Create(filePath)
+				Expect(err).Error().ShouldNot(HaveOccurred())
+				defer f.Close()
+				_, err = f.WriteString(content)
+				Expect(err).Error().ShouldNot(HaveOccurred())
+			}
+			// test multilevel dir
+			const subDir = "subdir/subsubdir"
+			filePath = filepath.Join(tempDir, subDir, "test.yml")
+			err := os.MkdirAll(filepath.Dir(filePath), 0755)
+			f, err := os.Create(filePath)
+			Expect(err).Error().ShouldNot(HaveOccurred())
+			defer f.Close()
+			_, err = f.WriteString(contents[len(contents)-1])
+		})
+		It("should return all contents", func() {
+			filePath = tempDir
+		})
+	})
+})

Original file line number	Diff line number	Diff line change
`@@ -33,7 +33,7 @@ func applyCMDFunc(cmd *cobra.Command, args []string) {`
`33`	`33`	`}`
`34`	`34`
`35`	`35`	`func init() {`
`36`		`- applyCMD.Flags().StringVarP(&configFilePath, configFlagName, "f", "config.yaml", "config file")`
	`36`	`+ applyCMD.Flags().StringVarP(&configFilePath, configFlagName, "f", "", "config file or directory")`
`37`	`37`	`applyCMD.Flags().StringVarP(&pluginDir, pluginDirFlagName, "d", defaultPluginDir, "plugins directory")`
`38`	`38`	`applyCMD.Flags().BoolVarP(&continueDirectly, "yes", "y", false, "apply directly without confirmation")`
`39`	`39`
Original file line number	Diff line number	Diff line change
`@@ -2,19 +2,20 @@ package configmanager`
`2`	`2`
`3`	`3`	`import (`
`4`	`4`	`"fmt"`
`5`		`- "os"`
`6`	`5`	`"strings"`
	`6`	`+`
	`7`	`+ "github.com/devstream-io/devstream/pkg/util/file"`
`7`	`8`	`)`
`8`	`9`
`9`	`10`	`// Manager is used to load the config file from the ConfigFilePath and finally get the Config object.`
`10`	`11`	`type Manager struct {`
`11`	`12`	`ConfigFilePath string`
`12`	`13`	`}`
`13`	`14`
`14`		`-// NewManager takes configFilePath, then return a *Manager object.`
`15`		`-func NewManager(configFilePath string) *Manager {`
	`15`	`+// NewManager takes configFilePath(file or directory), then return a *Manager object.`
	`16`	`+func NewManager(configPath string) *Manager {`
`16`	`17`	`return &Manager{`
`17`		`- ConfigFilePath: configFilePath,`
	`18`	`+ ConfigFilePath: configPath,`
`18`	`19`	`}`
`19`	`20`	`}`
`20`	`21`
`@@ -46,7 +47,7 @@ func (m Manager) LoadConfig() (Config, error) {`
`46`	`47`	`// 1. variables rendered must be before the yaml.Unmarshal() called for the [[ foo ]] will be treated as a two-dimensional array by the yaml parser;`
`47`	`48`	`// 2. the variables used([[ foo ]]) in the Config.PipelineTemplates can be defined in the Config.Apps or Config.Vars;`
`48`	`49`	`func (m Manager) getConfigFromFileWithGlobalVars() (Config, error) {`
`49`		`- configBytes, err := os.ReadFile(m.ConfigFilePath)`
	`50`	`+ configBytes, err := file.ReadYamls(m.ConfigFilePath)`
`50`	`51`	`if err != nil {`
`51`	`52`	`return nil, err`
`52`	`53`	`}`
Original file line number	Diff line number	Diff line change
`@@ -22,7 +22,7 @@ func (g *GitHubWorkflow) CIFilePath() string {`
`22`	`22`	`return filepath.Join(ciGitHubWorkConfigLocation, ciGithubTempName)`
`23`	`23`	`}`
`24`	`24`
`25`		`-func (g *GitHubWorkflow) FilterCIFilesFunc() file.DirFIleFilterFunc {`
	`25`	`+func (g *GitHubWorkflow) FilterCIFilesFunc() file.DirFileFilterFunc {`
`26`	`26`	`return func(filePath string, isDir bool) bool {`
`27`	`27`	`// not process dir`
`28`	`28`	`if isDir {`
Original file line number	Diff line number	Diff line change
`@@ -19,7 +19,7 @@ func (g *GitLabCI) CIFilePath() string {`
`19`	`19`	`return ciGitLabConfigLocation`
`20`	`20`	`}`
`21`	`21`
`22`		`-func (g *GitLabCI) FilterCIFilesFunc() file.DirFIleFilterFunc {`
	`22`	`+func (g *GitLabCI) FilterCIFilesFunc() file.DirFileFilterFunc {`
`23`	`23`	`return func(filePath string, isDir bool) bool {`
`24`	`24`	`// not process dir`
`25`	`25`	`if isDir {`
Original file line number	Diff line number	Diff line change
`@@ -20,7 +20,7 @@ func (j *JenkinsPipeline) CIFilePath() string {`
`20`	`20`	`}`
`21`	`21`
`22`	`22`	`// FilterCIFilesFunc only get file with name Jenkinsfile`
`23`		`-func (j *JenkinsPipeline) FilterCIFilesFunc() file.DirFIleFilterFunc {`
	`23`	`+func (j *JenkinsPipeline) FilterCIFilesFunc() file.DirFileFilterFunc {`
`24`	`24`	`return func(filePath string, isDir bool) bool {`
`25`	`25`	`// not process dir`
`26`	`26`	`if isDir {`
Original file line number	Diff line number	Diff line change
`@@ -15,7 +15,7 @@ type CIServerOptions interface {`
`15`	`15`	`CIFilePath() string`
`16`	`16`
`17`	`17`	`// FilterCIFilesFunc returns a filter function to select ci config file`
`18`		`- FilterCIFilesFunc() file.DirFIleFilterFunc`
	`18`	`+ FilterCIFilesFunc() file.DirFileFilterFunc`
`19`	`19`	`// GetGitNameFunc returns a function to transform file path to git name of ci config file`
`20`	`20`	`GetGitNameFunc() file.DirFileNameFunc`
`21`	`21`	`}`