Add postgres experience (#95)

* Initial implementation of flavors template registry.

Signed-off-by: Christoph Pakulski <[email protected]>

* Moved logic producing config from individual flavor to generic flavor package.

Signed-off-by: Christoph Pakulski <[email protected]>

* Added InPort listening port to template parameters.

Signed-off-by: Christoph Pakulski <[email protected]>

* Added comments and corrected formatting.

Signed-off-by: Christoph Pakulski <[email protected]>

* Format and style changes for linter and go format checker.

Signed-off-by: Christoph Pakulski <[email protected]>

* Code corrections for linter.

Signed-off-by: Christoph Pakulski <[email protected]>

* Reduced gocycle complexity in NewRunCmd.

Signed-off-by: Christoph Pakulski <[email protected]>

* Changes after code review.

Signed-off-by: Christoph Pakulski <[email protected]>

* Fixed crash when template args are specified and getnevoy args point directly to Envoy image.

Signed-off-by: Christoph Pakulski <[email protected]>

* Added logic to create static or strict_dns type of cluster
based on comma separated list of endpoints.

Signed-off-by: Christoph Pakulski <[email protected]>

* Added error checking when template.Execute functions are run
when postgres specific config is created out of templates.

Signed-off-by: Christoph Pakulski <[email protected]>

* Renamed template argument from endpoint to endpoints.

Signed-off-by: Christoph Pakulski <[email protected]>

* Corrected methods' comments. Small refactoring of unit tests.

Signed-off-by: Christoph Pakulski <[email protected]>

* Removed checking for user specified config params.

Signed-off-by: Christoph Pakulski <[email protected]>

Author: cpakulski

go.mod

@@ -5,6 +5,7 @@ go 1.12
 require (
 	bitbucket.org/creachadair/shell v0.0.6
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
+	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48
 	github.com/golang/protobuf v1.3.2

go.sum

@@ -50,6 +50,7 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
 github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
 github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.13.24/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=

pkg/binary/envoy/config.go

@@ -15,6 +15,10 @@
 package envoy
 
 import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/gogo/protobuf/types"
@@ -70,3 +74,18 @@ type Config struct {
 	AdminPort      int32
 	StatNameLength int32
 }
+
+// SaveConfig saves configuration string in getenvoy
+// directory.
+func (r *Runtime) SaveConfig(name, config string) (string, error) {
+	configDir := filepath.Join(r.RootDir, "configs")
+	if err := os.MkdirAll(configDir, 0750); err != nil {
+		return "", fmt.Errorf("Unable to create directory %q: %v", configDir, err)
+	}
+	filename := name + ".yaml"
+	err := ioutil.WriteFile(filepath.Join(configDir, filename), []byte(config), 0644)
+	if err != nil {
+		return "", fmt.Errorf("Cannot save config file %s: %s", filepath.Join(configDir, filename), err)
+	}
+	return filepath.Join(configDir, filename), nil
+}

pkg/binary/envoy/runtime.go

@@ -34,6 +34,7 @@ func NewRuntime(options ...func(*Runtime)) (binary.FetchRunner, error) {
 	local := filepath.Join(usrDir, ".getenvoy")
 	runtime := &Runtime{
 		Config:         NewConfig(),
+		RootDir:        local,
 		fetcher:        fetcher{local},
 		TmplDir:        filepath.Join(local, "templates"),
 		wg:             &sync.WaitGroup{},
@@ -60,6 +61,7 @@ type fetcher struct {
 type Runtime struct {
 	fetcher
 
+	RootDir  string
 	debugDir string
 	TmplDir  string
 	Config   *Config

pkg/binary/interface.go

@@ -39,6 +39,10 @@ type Runner interface {
 	DebugStore() string
 	SetStdout(w io.Writer)
 	SetStderr(w io.Writer)
+	// TODO(cpakulski) Maybe Runner is not the best place for SaveConfig.
+	// It was added here because it needs location of .getenvoy
+	// directory.
+	SaveConfig(name string, config string) (string, error)
 }
 
 const (

pkg/cmd/run.go

@@ -24,6 +24,8 @@ import (
 	"github.com/tetratelabs/getenvoy/pkg/binary/envoy"
 	"github.com/tetratelabs/getenvoy/pkg/binary/envoy/controlplane"
 	"github.com/tetratelabs/getenvoy/pkg/binary/envoy/debug"
+	"github.com/tetratelabs/getenvoy/pkg/flavors"
+	_ "github.com/tetratelabs/getenvoy/pkg/flavors/postgres" //nolint
 	"github.com/tetratelabs/getenvoy/pkg/manifest"
 )
 
@@ -32,6 +34,7 @@ var (
 	accessLogServerAddress string
 	mode                   string
 	bootstrap              string
+	templateArgs           map[string]string
 )
 
 // NewRunCmd create a command responsible for starting an Envoy process
@@ -53,18 +56,13 @@ getenvoy run ./envoy -- --config-path ./bootstrap.yaml
 
 # List available Envoy flags.
 getenvoy run standard:1.11.1 -- --help
+
+# Run with Postgres specific configuration bootstrapped
+getenvoy run postgres:nightly --templateArg endpoints=127.0.0.1:5432,192.168.0.101:5432 --templateArg inport=5555
 `,
 		Args: func(cmd *cobra.Command, args []string) error {
-			if len(args) == 0 {
-				return errors.New("missing binary parameter")
-			}
-			if err := validateMode(); err != nil {
-				return err
-			}
-			if err := validateBootstrap(); err != nil {
-				return err
-			}
-			return validateRequiresBootstrap()
+			return validateCmdArgs(args)
+
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			cfg := envoy.NewConfig(
@@ -88,12 +86,25 @@ getenvoy run standard:1.11.1 -- --help
 			}
 
 			key, manifestErr := manifest.NewKey(args[0])
+
 			if manifestErr != nil {
 				if _, err := os.Stat(args[0]); err != nil {
 					return fmt.Errorf("%v isn't valid manifest reference or an existing filepath", args[0])
 				}
 				return runtime.RunPath(args[0], args[1:])
 			}
+
+			// Check if the templateArgs were passed to the cmd line.
+			// If they were passed, config must be created based on
+			// template.
+			if len(templateArgs) > 0 {
+				cmdArg, err := processTemplateArgs(key.Flavor, templateArgs, runtime.(*envoy.Runtime))
+				if err != nil {
+					return err
+				}
+				args = append(args, cmdArg)
+			}
+
 			if !runtime.AlreadyDownloaded(key) {
 				location, err := manifest.Locate(key, manifestURL)
 				if err != nil {
@@ -114,6 +125,8 @@ getenvoy run standard:1.11.1 -- --help
 		"(experimental) location of Envoy's access log server <host|ip:port> (requires bootstrap flag)")
 	cmd.Flags().StringVar(&mode, "mode", "",
 		fmt.Sprintf("(experimental) mode to run Envoy in <%v> (requires bootstrap flag)", strings.Join(envoy.SupportedModes, "|")))
+	cmd.Flags().StringToStringVar(&templateArgs, "templateArg", map[string]string{},
+		"arguments passed to a config template for substitution")
 	return cmd
 }
 
@@ -153,6 +166,19 @@ func validateRequiresBootstrap() error {
 	return nil
 }
 
+func validateCmdArgs(args []string) error {
+	if len(args) == 0 {
+		return errors.New("missing binary parameter")
+	}
+	if err := validateMode(); err != nil {
+		return err
+	}
+	if err := validateBootstrap(); err != nil {
+		return err
+	}
+	return validateRequiresBootstrap()
+}
+
 func controlplaneFunc() func(r *envoy.Runtime) {
 	switch bootstrap {
 	case istio:
@@ -162,3 +188,18 @@ func controlplaneFunc() func(r *envoy.Runtime) {
 		return func(r *envoy.Runtime) {}
 	}
 }
+
+// Function creates config file based on template args passed by a user.
+// The return value is Envoy command line option which must be passed to Envoy.
+func processTemplateArgs(flavor string, templateArgs map[string]string, runtime *envoy.Runtime) (string, error) {
+	config, err := flavors.CreateConfig(flavor, templateArgs)
+	if err != nil {
+		return "", err
+	}
+	// Save config in getenvoy directory
+	path, err := runtime.SaveConfig(flavor, config)
+	if err != nil {
+		return "", err
+	}
+	return "--config-path " + path, nil
+}

pkg/flavors/flavors.go

@@ -0,0 +1,62 @@
+// Copyright 2020 Tetrate
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package flavors
+
+import (
+	"fmt"
+)
+
+// FlavorConfigTemplate represents interface to individual flavors.
+type FlavorConfigTemplate interface {
+	GenerateConfig(params map[string]string) (string, error)
+}
+
+// Main repo for templates.
+type flavorStore struct {
+	// This is a map indexed by flavor pointing to the individual
+	// implementaions of each flavor.
+	templates map[string]FlavorConfigTemplate
+}
+
+var store = flavorStore{templates: make(map[string]FlavorConfigTemplate)}
+
+// AddFlavor function is used by individual flavors (like postgres)
+// to add the flavor to main repo.
+func AddFlavor(flavor string, configTemplate FlavorConfigTemplate) {
+	store.templates[flavor] = configTemplate
+}
+
+// GetFlavor function returns FlavorConfigTemplate structure associated
+// with flavor.
+func GetFlavor(flavor string) (FlavorConfigTemplate, error) {
+	tmplString, ok := store.templates[flavor]
+	if !ok {
+		return nil, fmt.Errorf("Cannot find template for flavor %s", flavor)
+	}
+
+	return tmplString, nil
+}
+
+// CreateConfig function checks flavor specific parameters, get flavor's template and
+// create a config.
+func CreateConfig(flavor string, params map[string]string) (string, error) {
+	flavorData, err := GetFlavor(flavor)
+
+	if err != nil {
+		return "", err
+	}
+
+	return flavorData.GenerateConfig(params)
+}

pkg/flavors/flavors_test.go

@@ -0,0 +1,76 @@
+// Copyright 2020 Tetrate
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package flavors_test
+
+import (
+	"testing"
+
+	"github.com/tetratelabs/getenvoy/pkg/flavors"
+)
+
+// Flavor for mocking and testing.
+type TestFlavor struct {
+	Test string
+}
+
+var flavor TestFlavor
+
+const testConfig string = "This is UnitTest config"
+
+func (f *TestFlavor) GenerateConfig(params map[string]string) (string, error) {
+	// Just return predefined value.
+	// Normally a flavor would use template but
+	// here using templates is not part of the test.
+	return testConfig, nil
+}
+
+// Test adding and retrieving config template
+func TestAdd(t *testing.T) {
+	flavors.AddFlavor("test", &flavor)
+
+	_, err := flavors.GetFlavor("test")
+
+	if err != nil {
+		t.Error("Just added template cannot be located")
+	}
+}
+
+// Test retrieving non-existing template
+func TestGetNonExisting(t *testing.T) {
+	flavors.AddFlavor("test", &flavor)
+
+	_, err := flavors.GetFlavor("test1")
+
+	if err == nil {
+		t.Error("Error should be returned for non-existing template")
+	}
+}
+
+// Test creating config.
+// Test verifies that after adding TestFlavor to the list
+// of known flavors, it can create a proper config.
+func TestCreateConfig(t *testing.T) {
+	flavors.AddFlavor("test", &flavor)
+
+	params := map[string]string{"Test": "UnitTest"}
+	config, err := flavors.CreateConfig("test", params)
+
+	if err != nil {
+		t.Error("Creating config failed with proper parameters")
+	}
+
+	if config != testConfig {
+		t.Errorf("Created config %s not as expected", config)
+	}
+}