feat: implement config setup for each component in bare-metal mode (#153)

shawnh2 · web-flow · commit 4394eab82a1e · 2023-09-22T12:45:44.000+08:00
* add merge operation for bare-metal config

Signed-off-by: sh2 &lt;shawnhxh@outlook.com&gt;

* add custom configuration file support for each component in bare-metal mode

Signed-off-by: sh2 &lt;shawnhxh@outlook.com&gt;

---------

Signed-off-by: sh2 &lt;shawnhxh@outlook.com&gt;
diff --git a/examples/bare-metal/cluster-with-s3-storage.datanode.toml b/examples/bare-metal/cluster-with-s3-storage.datanode.toml
@@ -0,0 +1,8 @@
+# More options for storage: https://docs.greptime.com/user-guide/operations/configuration#storage-options
+
+[storage]
+type = "S3"
+bucket = "test_greptimedb"
+root = "/greptimedb"
+access_key_id = "<access key id>"
+secret_access_key = "<secret access key>"
diff --git a/examples/bare-metal/cluster-with-s3-storage.yaml b/examples/bare-metal/cluster-with-s3-storage.yaml
@@ -0,0 +1,21 @@
+cluster:
+  name: mycluster # name of the cluster
+  artifact:
+    version: latest
+  frontend:
+    replicas: 1
+  datanode:
+    replicas: 3
+    rpcAddr: 0.0.0.0:14100
+    mysqlAddr: 0.0.0.0:14200
+    httpAddr: 0.0.0.0:14300
+    config: 'examples/bare-metal/cluster-with-s3-storage.datanode.toml'
+  meta:
+    replicas: 1
+    storeAddr: 127.0.0.1:2379
+    serverAddr: 0.0.0.0:3002
+    httpAddr: 0.0.0.0:14001
+
+etcd:
+  artifact:
+    version: v3.5.7
diff --git a/go.mod b/go.mod
@@ -19,6 +19,7 @@ require (
 	github.com/onsi/gomega v1.23.0
 	github.com/spf13/cobra v1.6.1
 	github.com/stretchr/testify v1.8.2
+	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
 	gopkg.in/yaml.v3 v3.0.1
 	helm.sh/helm/v3 v3.11.1
 	k8s.io/api v0.26.0
@@ -137,7 +138,7 @@ require (
 	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/oauth2 v0.8.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.11.0 // indirect
+	golang.org/x/sys v0.12.0 // indirect
 	golang.org/x/term v0.11.0 // indirect
 	golang.org/x/text v0.12.0 // indirect
 	golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
diff --git a/go.sum b/go.sum
@@ -674,6 +674,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
+golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -852,8 +854,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
-golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
diff --git a/pkg/cmd/gtctl/cluster/create/create.go b/pkg/cmd/gtctl/cluster/create/create.go
@@ -180,7 +180,7 @@ func NewCluster(args []string, options ClusterCliOptions, l logger.Logger) error
 	}
 
 	if options.BareMetal {
-		if err := waitChildProcess(ctx, clusterDeployer, false, deleteOpts); err != nil {
+		if err = waitChildProcess(ctx, clusterDeployer, false, deleteOpts); err != nil {
 			return err
 		}
 	}
@@ -205,17 +205,17 @@ func newDeployer(l logger.Logger, clusterName string, options *ClusterCliOptions
 	}
 
 	if options.Config != "" {
-		var config bmconfig.Config
-		data, err := os.ReadFile(options.Config)
+		var cfg bmconfig.Config
+		raw, err := os.ReadFile(options.Config)
 		if err != nil {
 			return nil, err
 		}
 
-		if err := yaml.Unmarshal(data, &config); err != nil {
+		if err = yaml.Unmarshal(raw, &cfg); err != nil {
 			return nil, err
 		}
 
-		opts = append(opts, baremetal.WithConfig(&config))
+		opts = append(opts, baremetal.WithMergeConfig(&cfg, raw))
 	}
 
 	opts = append(opts, baremetal.WithAlawaysDownload(options.AlwaysDownload))
@@ -358,6 +358,7 @@ func waitChildProcess(ctx context.Context, deployer deployer.Interface, close bo
 			fmt.Printf("\x1b[32m%s\x1b[0m", fmt.Sprintf("To view dashboard by accessing: %s\n", logger.Bold("http://localhost:4000/dashboard/")))
 		} else {
 			fmt.Printf("\x1b[32m%s\x1b[0m", fmt.Sprintf("The cluster(pid=%d, version=%s) run in bare-metal has been deleted now...\n", os.Getpid(), v))
+			return nil
 		}
 
 		// Wait for all the child processes to exit.
diff --git a/pkg/deployer/baremetal/component/datanode.go b/pkg/deployer/baremetal/component/datanode.go
@@ -134,6 +134,11 @@ func (d *datanode) BuildArgs(ctx context.Context, params ...interface{}) []strin
 		fmt.Sprintf("--data-home=%s", dataHomeDir),
 		fmt.Sprintf("--wal-dir=%s", walDir),
 	}
+
+	if len(d.config.Config) > 0 {
+		args = append(args, fmt.Sprintf("-c %s", d.config.Config))
+	}
+
 	return args
 }
 
diff --git a/pkg/deployer/baremetal/component/frontend.go b/pkg/deployer/baremetal/component/frontend.go
@@ -87,11 +87,17 @@ func (f *frontend) BuildArgs(ctx context.Context, params ...interface{}) []strin
 	if logLevel == "" {
 		logLevel = config.DefaultLogLevel
 	}
+
 	args := []string{
 		fmt.Sprintf("--log-level=%s", logLevel),
 		f.Name(), "start",
 		fmt.Sprintf("--metasrv-addr=%s", f.metaSrvAddr),
 	}
+
+	if len(f.config.Config) > 0 {
+		args = append(args, fmt.Sprintf("-c %s", f.config.Config))
+	}
+
 	return args
 }
 
diff --git a/pkg/deployer/baremetal/component/metasrv.go b/pkg/deployer/baremetal/component/metasrv.go
@@ -119,11 +119,16 @@ func (m *metaSrv) BuildArgs(ctx context.Context, params ...interface{}) []string
 	args := []string{
 		fmt.Sprintf("--log-level=%s", logLevel),
 		m.Name(), "start",
-		"--store-addr", m.config.StoreAddr,
-		"--server-addr", m.config.ServerAddr,
-		"--http-addr", generateMetaSrvAddr(m.config.HTTPAddr, nodeID),
-		"--bind-addr", generateMetaSrvAddr(bindAddr, nodeID),
+		fmt.Sprintf("--store-addr=%s", m.config.StoreAddr),
+		fmt.Sprintf("--server-addr=%s", m.config.ServerAddr),
+		fmt.Sprintf("--http-addr=%s", generateMetaSrvAddr(m.config.HTTPAddr, nodeID)),
+		fmt.Sprintf("--bind-addr=%s", generateMetaSrvAddr(bindAddr, nodeID)),
 	}
+
+	if len(m.config.Config) > 0 {
+		args = append(args, fmt.Sprintf("-c %s", m.config.Config))
+	}
+
 	return args
 }
 
diff --git a/pkg/deployer/baremetal/config/datanode.go b/pkg/deployer/baremetal/config/datanode.go
@@ -24,6 +24,7 @@ type Datanode struct {
 	DataDir      string `yaml:"dataDir" validate:"omitempty,dirpath"`
 	WalDir       string `yaml:"walDir" validate:"omitempty,dirpath"`
 	ProcedureDir string `yaml:"procedureDir" validate:"omitempty,dirpath"`
+	Config       string `yaml:"config" validate:"omitempty,filepath"`
 
 	LogLevel string `yaml:"logLevel"`
 }
diff --git a/pkg/deployer/baremetal/config/frontend.go b/pkg/deployer/baremetal/config/frontend.go
@@ -22,5 +22,7 @@ type Frontend struct {
 	PostgresAddr string `yaml:"postgresAddr" validate:"omitempty,hostname_port"`
 	MetaAddr     string `yaml:"metaAddr" validate:"omitempty,hostname_port"`
 
+	Config string `yaml:"config" validate:"omitempty,filepath"`
+
 	LogLevel string `yaml:"logLevel"`
 }
diff --git a/pkg/deployer/baremetal/config/metasrv.go b/pkg/deployer/baremetal/config/metasrv.go
@@ -22,5 +22,7 @@ type MetaSrv struct {
 	BindAddr   string `yaml:"bindAddr" validate:"omitempty,hostname_port"`
 	HTTPAddr   string `yaml:"httpAddr" validate:"required,hostname_port"`
 
+	Config string `yaml:"config" validate:"omitempty,filepath"`
+
 	LogLevel string `yaml:"logLevel"`
 }
diff --git a/pkg/deployer/baremetal/deployer.go b/pkg/deployer/baremetal/deployer.go
@@ -96,14 +96,14 @@ func NewDeployer(l logger.Logger, clusterName string, opts ...Option) (Interface
 	d.initClusterDirsAndPath(clusterName)
 
 	if !d.createNoDirs {
-		if err := d.createClusterDirs(); err != nil {
+		if err = d.createClusterDirs(); err != nil {
 			return nil, err
 		}
 
 		d.bm = component.NewBareMetalCluster(d.config.Cluster, d.workingDirs, &d.wg, d.logger)
 
 		// Save a copy of cluster config in yaml format.
-		if err := d.createClusterConfigFile(); err != nil {
+		if err = d.createClusterConfigFile(); err != nil {
 			return nil, err
 		}
 	}
@@ -188,10 +188,40 @@ func (d *Deployer) createClusterConfigFile() error {
 	return nil
 }
 
-func WithConfig(config *config.Config) Option {
-	// TODO(zyy17): Should merge the default configuration.
+// WithMergeConfig merges config with current deployer config.
+// It will perform WithReplaceConfig if any error occurs during merging or receive nil raw config.
+func WithMergeConfig(cfg *config.Config, rawConfig []byte) Option {
+	if len(rawConfig) == 0 {
+		return WithReplaceConfig(cfg)
+	}
+
+	return func(d *Deployer) {
+		defaultConfig, err := yaml.Marshal(d.config)
+		if err != nil {
+			d.config = cfg
+			return
+		}
+
+		out, err := fileutils.MergeYAML(defaultConfig, rawConfig)
+		if err != nil {
+			d.config = cfg
+			return
+		}
+
+		var newConfig config.Config
+		if err = yaml.Unmarshal(out, &newConfig); err != nil {
+			d.config = cfg
+			return
+		}
+
+		d.config = &newConfig
+	}
+}
+
+// WithReplaceConfig replaces config with current deployer config.
+func WithReplaceConfig(cfg *config.Config) Option {
 	return func(d *Deployer) {
-		d.config = config
+		d.config = cfg
 	}
 }
 
diff --git a/pkg/deployer/baremetal/deployer_test.go b/pkg/deployer/baremetal/deployer_test.go
@@ -61,10 +61,10 @@ func TestNewDeployer(t *testing.T) {
 	assert.NotNil(t, d2)
 	assert.True(t, d2.alwaysDownload)
 
-	// New Deployer with config option
+	// New Deployer with replace config option
 	newConfig := config.DefaultConfig()
 	newConfig.Cluster.Frontend.Replicas = 3
-	deployer, err = NewDeployer(L, clusterName, WithConfig(newConfig))
+	deployer, err = NewDeployer(L, clusterName, WithReplaceConfig(newConfig))
 	assert.NoError(t, err)
 	d3, ok := deployer.(*Deployer)
 	assert.True(t, ok)
diff --git a/pkg/utils/file/yaml.go b/pkg/utils/file/yaml.go
@@ -0,0 +1,52 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package file
+
+import (
+	"bytes"
+
+	"golang.org/x/exp/maps"
+	"gopkg.in/yaml.v3"
+)
+
+// MergeYAML merges two yaml files from src to dst, the src yaml will override dst yaml if the same key exists.
+func MergeYAML(dst, src []byte) ([]byte, error) {
+	map1 := map[string]interface{}{}
+	map2 := map[string]interface{}{}
+
+	if err := yaml.Unmarshal(src, &map1); err != nil {
+		return nil, err
+	}
+
+	if err := yaml.Unmarshal(dst, &map2); err != nil {
+		return nil, err
+	}
+
+	maps.Copy(map2, map1)
+
+	buf := bytes.NewBuffer([]byte{})
+	encoder := yaml.NewEncoder(buf)
+	encoder.SetIndent(2)
+
+	if err := encoder.Encode(map2); err != nil {
+		return nil, err
+	}
+
+	if err := encoder.Close(); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
diff --git a/pkg/utils/file/yaml_test.go b/pkg/utils/file/yaml_test.go
@@ -0,0 +1,80 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package file
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestMergeYAML(t *testing.T) {
+	tests := []struct {
+		name  string
+		yaml1 string
+		yaml2 string
+		want  string
+	}{
+		{
+			name: "test",
+			yaml1: `
+a: a
+b: 
+  c:
+    d: d
+e:
+  f:
+    - g
+k:
+  l: l
+`,
+			yaml2: `
+a: a1
+b: 
+  c:
+    d: d1
+e:
+  f:
+    - h
+i:
+  j: j
+`,
+			want: `a: a1
+b:
+  c:
+    d: d1
+e:
+  f:
+    - h
+i:
+  j: j
+k:
+  l: l
+`,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := MergeYAML([]byte(tt.yaml1), []byte(tt.yaml2))
+			if err != nil {
+				t.Errorf("MergeYAML() err = %v", err)
+			}
+
+			actual := string(got)
+			if !reflect.DeepEqual(actual, tt.want) {
+				t.Errorf("MergeYAML() got = %v, want %v", actual, tt.want)
+			}
+		})
+	}
+}

Original file line number	Diff line number	Diff line change
`@@ -180,7 +180,7 @@ func NewCluster(args []string, options ClusterCliOptions, l logger.Logger) error`
`180`	`180`	`}`
`181`	`181`
`182`	`182`	`if options.BareMetal {`
`183`		`- if err := waitChildProcess(ctx, clusterDeployer, false, deleteOpts); err != nil {`
	`183`	`+ if err = waitChildProcess(ctx, clusterDeployer, false, deleteOpts); err != nil {`
`184`	`184`	`return err`
`185`	`185`	`}`
`186`	`186`	`}`
`@@ -205,17 +205,17 @@ func newDeployer(l logger.Logger, clusterName string, options *ClusterCliOptions`
`205`	`205`	`}`
`206`	`206`
`207`	`207`	`if options.Config != "" {`
`208`		`- var config bmconfig.Config`
`209`		`- data, err := os.ReadFile(options.Config)`
	`208`	`+ var cfg bmconfig.Config`
	`209`	`+ raw, err := os.ReadFile(options.Config)`
`210`	`210`	`if err != nil {`
`211`	`211`	`return nil, err`
`212`	`212`	`}`
`213`	`213`
`214`		`- if err := yaml.Unmarshal(data, &config); err != nil {`
	`214`	`+ if err = yaml.Unmarshal(raw, &cfg); err != nil {`
`215`	`215`	`return nil, err`
`216`	`216`	`}`
`217`	`217`
`218`		`- opts = append(opts, baremetal.WithConfig(&config))`
	`218`	`+ opts = append(opts, baremetal.WithMergeConfig(&cfg, raw))`
`219`	`219`	`}`
`220`	`220`
`221`	`221`	`opts = append(opts, baremetal.WithAlawaysDownload(options.AlwaysDownload))`
`@@ -358,6 +358,7 @@ func waitChildProcess(ctx context.Context, deployer deployer.Interface, close bo`
`358`	`358`	`fmt.Printf("\x1b[32m%s\x1b[0m", fmt.Sprintf("To view dashboard by accessing: %s\n", logger.Bold("http://localhost:4000/dashboard/")))`
`359`	`359`	`} else {`
`360`	`360`	`fmt.Printf("\x1b[32m%s\x1b[0m", fmt.Sprintf("The cluster(pid=%d, version=%s) run in bare-metal has been deleted now...\n", os.Getpid(), v))`
	`361`	`+ return nil`
`361`	`362`	`}`
`362`	`363`
`363`	`364`	`// Wait for all the child processes to exit.`
Original file line number	Diff line number	Diff line change
`@@ -134,6 +134,11 @@ func (d *datanode) BuildArgs(ctx context.Context, params ...interface{}) []strin`
`134`	`134`	`fmt.Sprintf("--data-home=%s", dataHomeDir),`
`135`	`135`	`fmt.Sprintf("--wal-dir=%s", walDir),`
`136`	`136`	`}`
	`137`	`+`
	`138`	`+ if len(d.config.Config) > 0 {`
	`139`	`+ args = append(args, fmt.Sprintf("-c %s", d.config.Config))`
	`140`	`+ }`
	`141`	`+`
`137`	`142`	`return args`
`138`	`143`	`}`
`139`	`144`
Original file line number	Diff line number	Diff line change
`@@ -87,11 +87,17 @@ func (f *frontend) BuildArgs(ctx context.Context, params ...interface{}) []strin`
`87`	`87`	`if logLevel == "" {`
`88`	`88`	`logLevel = config.DefaultLogLevel`
`89`	`89`	`}`
	`90`	`+`
`90`	`91`	`args := []string{`
`91`	`92`	`fmt.Sprintf("--log-level=%s", logLevel),`
`92`	`93`	`f.Name(), "start",`
`93`	`94`	`fmt.Sprintf("--metasrv-addr=%s", f.metaSrvAddr),`
`94`	`95`	`}`
	`96`	`+`
	`97`	`+ if len(f.config.Config) > 0 {`
	`98`	`+ args = append(args, fmt.Sprintf("-c %s", f.config.Config))`
	`99`	`+ }`
	`100`	`+`
`95`	`101`	`return args`
`96`	`102`	`}`
`97`	`103`
Original file line number	Diff line number	Diff line change
`@@ -24,6 +24,7 @@ type Datanode struct {`
`24`	`24`	DataDir string `yaml:"dataDir" validate:"omitempty,dirpath"`
`25`	`25`	WalDir string `yaml:"walDir" validate:"omitempty,dirpath"`
`26`	`26`	ProcedureDir string `yaml:"procedureDir" validate:"omitempty,dirpath"`
	`27`	+ Config string `yaml:"config" validate:"omitempty,filepath"`
`27`	`28`
`28`	`29`	LogLevel string `yaml:"logLevel"`
`29`	`30`	`}`
Original file line number	Diff line number	Diff line change
`@@ -22,5 +22,7 @@ type Frontend struct {`
`22`	`22`	PostgresAddr string `yaml:"postgresAddr" validate:"omitempty,hostname_port"`
`23`	`23`	MetaAddr string `yaml:"metaAddr" validate:"omitempty,hostname_port"`
`24`	`24`
	`25`	+ Config string `yaml:"config" validate:"omitempty,filepath"`
	`26`	`+`
`25`	`27`	LogLevel string `yaml:"logLevel"`
`26`	`28`	`}`