Skip to content

Commit adc4555

Browse files
DmitryRomanovDmitryRomanov
committed
use many config files
1 parent de8f0fc commit adc4555

File tree

8 files changed

+199
-18
lines changed

8 files changed

+199
-18
lines changed

‎cfg/config.go‎

Lines changed: 44 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ import (
1313

1414
"github.com/bitly/go-simplejson"
1515
"github.com/ozontech/file.d/logger"
16-
"sigs.k8s.io/yaml"
16+
"gopkg.in/yaml.v2"
17+
k8s_yaml "sigs.k8s.io/yaml"
1718
)
1819

1920
const trueValue = "true"
@@ -77,22 +78,37 @@ func NewConfig() *Config {
7778
}
7879
}
7980

80-
func NewConfigFromFile(path string) *Config {
81-
logger.Infof("reading config %q", path)
82-
yamlContents, err := os.ReadFile(path)
81+
func NewConfigFromFile(paths []string) *Config {
82+
mergedConfig := make(map[interface{}]interface{})
83+
84+
for _, path := range paths {
85+
logger.Infof("reading config %q", path)
86+
yamlContents, err := os.ReadFile(path)
87+
if err != nil {
88+
logger.Fatalf("can't read config file %q: %s", path, err)
89+
}
90+
var currentConfig map[interface{}]interface{}
91+
if err := yaml.Unmarshal(yamlContents, &currentConfig); err != nil {
92+
logger.Fatalf("can't parse config file yaml %q: %s", path, err)
93+
}
94+
95+
mergedConfig = mergeYAMLs(mergedConfig, currentConfig)
96+
}
97+
98+
mergedYAML, err := yaml.Marshal(mergedConfig)
8399
if err != nil {
84-
logger.Fatalf("can't read config file %q: %s", path, err)
100+
logger.Fatalf("can't marshal merged config to YAML: %s", err)
85101
}
86102

87-
jsonContents, err := yaml.YAMLToJSON(yamlContents)
103+
jsonContents, err := k8s_yaml.YAMLToJSON(mergedYAML)
88104
if err != nil {
89-
logger.Infof("config content:\n%s", logger.Numerate(string(yamlContents)))
90-
logger.Fatalf("can't parse config file yaml %q: %s", path, err.Error())
105+
logger.Infof("config content:\n%s", logger.Numerate(string(mergedYAML)))
106+
logger.Fatalf("can't parse config file yaml %q: %s", paths, err.Error())
91107
}
92108

93109
object, err := simplejson.NewJson(jsonContents)
94110
if err != nil {
95-
logger.Fatalf("can't convert config to json %q: %s", path, err.Error())
111+
logger.Fatalf("can't convert config to json %q: %s", paths, err.Error())
96112
}
97113

98114
err = applyEnvs(object)
@@ -637,3 +653,22 @@ func CompileRegex(s string) (*regexp.Regexp, error) {
637653

638654
return regexp.Compile(s[1 : len(s)-1])
639655
}
656+
657+
func mergeYAMLs(a, b map[interface{}]interface{}) map[interface{}]interface{} {
658+
merged := make(map[interface{}]interface{})
659+
for k, v := range a {
660+
merged[k] = v
661+
}
662+
for k, v := range b {
663+
if existingValue, exists := merged[k]; exists {
664+
if existingMap, ok := existingValue.(map[interface{}]interface{}); ok {
665+
if newMap, ok := v.(map[interface{}]interface{}); ok {
666+
merged[k] = mergeYAMLs(existingMap, newMap)
667+
continue
668+
}
669+
}
670+
}
671+
merged[k] = v
672+
}
673+
return merged
674+
}

‎cfg/config_test.go‎

Lines changed: 145 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package cfg
33
import (
44
"encoding/json"
55
"errors"
6+
"reflect"
67
"testing"
78
"time"
89

@@ -11,16 +12,24 @@ import (
1112
"github.com/stretchr/testify/require"
1213
)
1314

14-
func NewTestConfig(name string) *Config {
15-
return NewConfigFromFile("../testdata/config/" + name)
15+
func NewTestConfig(names []string) *Config {
16+
var configFiles []string
17+
for _, name := range names {
18+
configFiles = append(configFiles, "../testdata/config/"+name)
19+
}
20+
return NewConfigFromFile(configFiles)
1621
}
1722

1823
func TestSimple(t *testing.T) {
19-
c := NewTestConfig("e2e.yaml")
24+
c := NewTestConfig([]string{"e2e.yaml", "e2e.override.yaml"})
2025

2126
assert.NotNil(t, c, "config loading should't return nil")
22-
2327
assert.Equal(t, 1, len(c.Pipelines), "pipelines count isn't match")
28+
29+
// check override config
30+
outputType, err := c.Pipelines["test"].Raw.Get("output").Get("type").String()
31+
assert.Nil(t, err, "cannot get output type")
32+
assert.Equal(t, "devnull", outputType, "output type is not overrided")
2433
}
2534

2635
type intDefault struct {
@@ -643,3 +652,135 @@ func TestExpression_UnmarshalJSON(t *testing.T) {
643652
require.Equal(t, Expression("2"), val.E2)
644653
require.Equal(t, Expression("2+2"), val.E3)
645654
}
655+
656+
func TestMergeYAMLs(t *testing.T) {
657+
tests := []struct {
658+
name string
659+
a map[interface{}]interface{}
660+
b map[interface{}]interface{}
661+
expected map[interface{}]interface{}
662+
}{
663+
{
664+
name: "simple merge",
665+
a: map[interface{}]interface{}{
666+
"key1": "value1",
667+
"key2": "value2",
668+
},
669+
b: map[interface{}]interface{}{
670+
"key2": "newValue2",
671+
"key3": "value3",
672+
},
673+
expected: map[interface{}]interface{}{
674+
"key1": "value1",
675+
"key2": "newValue2",
676+
"key3": "value3",
677+
},
678+
},
679+
{
680+
name: "nested maps",
681+
a: map[interface{}]interface{}{
682+
"key1": map[interface{}]interface{}{
683+
"subkey1": "subvalue1",
684+
},
685+
},
686+
b: map[interface{}]interface{}{
687+
"key1": map[interface{}]interface{}{
688+
"subkey2": "subvalue2",
689+
},
690+
"key2": "value2",
691+
},
692+
expected: map[interface{}]interface{}{
693+
"key1": map[interface{}]interface{}{
694+
"subkey1": "subvalue1",
695+
"subkey2": "subvalue2",
696+
},
697+
"key2": "value2",
698+
},
699+
},
700+
{
701+
name: "overwriting nested maps",
702+
a: map[interface{}]interface{}{
703+
"key1": map[interface{}]interface{}{
704+
"subkey1": "subvalue1",
705+
},
706+
},
707+
b: map[interface{}]interface{}{
708+
"key1": map[interface{}]interface{}{
709+
"subkey1": "newSubvalue1",
710+
"subkey2": "subvalue2",
711+
},
712+
},
713+
expected: map[interface{}]interface{}{
714+
"key1": map[interface{}]interface{}{
715+
"subkey1": "newSubvalue1",
716+
"subkey2": "subvalue2",
717+
},
718+
},
719+
},
720+
{
721+
name: "empty maps",
722+
a: map[interface{}]interface{}{},
723+
b: map[interface{}]interface{}{},
724+
expected: map[interface{}]interface{}{
725+
// Expecting an empty map
726+
},
727+
},
728+
{
729+
name: "a is empty",
730+
a: map[interface{}]interface{}{},
731+
b: map[interface{}]interface{}{
732+
"key1": "value1",
733+
},
734+
expected: map[interface{}]interface{}{
735+
"key1": "value1",
736+
},
737+
},
738+
{
739+
name: "b is empty",
740+
a: map[interface{}]interface{}{
741+
"key1": "value1",
742+
},
743+
b: map[interface{}]interface{}{},
744+
expected: map[interface{}]interface{}{
745+
"key1": "value1",
746+
},
747+
},
748+
{
749+
name: "override slice",
750+
a: map[interface{}]interface{}{
751+
"key1": []interface{}{"value1", "value2"},
752+
},
753+
b: map[interface{}]interface{}{
754+
"key1": []interface{}{"newValue1", "newValue2"},
755+
},
756+
expected: map[interface{}]interface{}{
757+
"key1": []interface{}{"newValue1", "newValue2"},
758+
},
759+
},
760+
{
761+
name: "merge slice with map",
762+
a: map[interface{}]interface{}{
763+
"key1": []interface{}{"value1", "value2"},
764+
},
765+
b: map[interface{}]interface{}{
766+
"key1": map[interface{}]interface{}{
767+
"subkey1": "subvalue1",
768+
},
769+
},
770+
expected: map[interface{}]interface{}{
771+
"key1": map[interface{}]interface{}{
772+
"subkey1": "subvalue1",
773+
},
774+
},
775+
},
776+
}
777+
778+
for _, tt := range tests {
779+
t.Run(tt.name, func(t *testing.T) {
780+
result := mergeYAMLs(tt.a, tt.b)
781+
if !reflect.DeepEqual(result, tt.expected) {
782+
t.Errorf("expected %v, got %v", tt.expected, result)
783+
}
784+
})
785+
}
786+
}

‎cmd/file.d/file.d.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ var (
6565
fileD *fd.FileD
6666
exit = make(chan bool)
6767

68-
config = kingpin.Flag("config", `Config file name`).Required().ExistingFile()
68+
config = kingpin.Flag("config", `Config file name (to add a config file you can repeat the argument)`).Required().ExistingFiles()
6969
http = kingpin.Flag("http", `HTTP listen addr eg. ":9000", "off" to disable`).Default(":9000").String()
7070
memLimitRatio = kingpin.Flag(
7171
"mem-limit-ratio",

‎cmd/file.d/file.d_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ const testTime = 10 * time.Minute
7070
// E.g. keep this test running while you are sleeping :)
7171
func TestEndToEnd(t *testing.T) {
7272
configFilename := "./../testdata/config/e2e.yaml"
73+
configOverrideFilename := "./../testdata/config/e2e.override.yaml"
7374
iterationInterval := time.Second * 10
7475
writerCount := 8
7576
fileCount := 8
@@ -85,7 +86,7 @@ func TestEndToEnd(t *testing.T) {
8586
filesDir := t.TempDir()
8687
offsetsDir := t.TempDir()
8788

88-
config := cfg.NewConfigFromFile(configFilename)
89+
config := cfg.NewConfigFromFile([]string{configFilename, configOverrideFilename})
8990
input := config.Pipelines["test"].Raw.Get("input")
9091
input.Set("watching_dir", filesDir)
9192
input.Set("offsets_file", filepath.Join(offsetsDir, "offsets.yaml"))

‎e2e/start_work_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ func TestE2EStabilityWorkCase(t *testing.T) {
162162
}
163163

164164
func startForTest(t *testing.T, test E2ETest, num int) *fd.FileD {
165-
conf := cfg.NewConfigFromFile(test.cfgPath)
165+
conf := cfg.NewConfigFromFile([]string{test.cfgPath})
166166
if _, ok := conf.Pipelines[test.name]; !ok {
167167
log.Fatalf("pipeline name must be named the same as the name of the test")
168168
}

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ require (
4848
go.uber.org/zap v1.25.0
4949
golang.org/x/net v0.21.0
5050
google.golang.org/protobuf v1.33.1-0.20240408130810-98873a205002
51+
gopkg.in/yaml.v2 v2.4.0
5152
gopkg.in/yaml.v3 v3.0.1
5253
k8s.io/api v0.27.4
5354
k8s.io/apimachinery v0.27.4
@@ -149,7 +150,6 @@ require (
149150
google.golang.org/appengine v1.6.7 // indirect
150151
gopkg.in/inf.v0 v0.9.1 // indirect
151152
gopkg.in/ini.v1 v1.62.0 // indirect
152-
gopkg.in/yaml.v2 v2.4.0 // indirect
153153
k8s.io/klog/v2 v2.90.1 // indirect
154154
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
155155
k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect

‎testdata/config/e2e.override.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
pipelines:
2+
test:
3+
output:
4+
type: devnull

‎testdata/config/e2e.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,4 @@ pipelines:
2626
# metric_name: throttle
2727
# metric_labels: [service, k8s_pod, k8s_container]
2828
output:
29-
type: devnull
29+
type: stdout

0 commit comments

Comments
 (0)