Fix data-gatherers config loading (#119)

* Fix data-gatherers config loading

Signed-off-by: Jose Fuentes <[email protected]>

* Rework unmarshaling of Config

Signed-off-by: Jose Fuentes <[email protected]>

Author: j-fuentes

agent.yaml

@@ -7,3 +7,5 @@ endpoint:
 data-gatherers:
   - kind: "dummy"
     name: "dummy"
+    config:
+      param-1: "bar"

pkg/agent/config.go

@@ -5,6 +5,12 @@ import (
 	"strings"
 
 	"github.com/hashicorp/go-multierror"
+	"github.com/jetstack/preflight/pkg/datagatherer"
+	"github.com/jetstack/preflight/pkg/datagatherer/aks"
+	"github.com/jetstack/preflight/pkg/datagatherer/eks"
+	"github.com/jetstack/preflight/pkg/datagatherer/gke"
+	"github.com/jetstack/preflight/pkg/datagatherer/k8s"
+	"github.com/jetstack/preflight/pkg/datagatherer/local"
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v2"
 )
@@ -25,10 +31,72 @@ type Endpoint struct {
 }
 
 type dataGatherer struct {
-	Kind     string `yaml:"kind"`
-	Name     string `yaml:"name"`
-	DataPath string `yaml:"data-path,omitempty"`
-	Config   []byte `yaml:"config"`
+	Kind     string
+	Name     string
+	DataPath string
+	Config   datagatherer.Config
+}
+
+func reMarshal(rawConfig interface{}, config datagatherer.Config) error {
+	bb, err := yaml.Marshal(rawConfig)
+	if err != nil {
+		return nil
+	}
+
+	err = yaml.Unmarshal(bb, config)
+	if err != nil {
+		return nil
+	}
+
+	return nil
+}
+
+// UnmarshalYAML unmarshals a dataGatherer resolving the type according to Kind.
+func (dg *dataGatherer) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	aux := struct {
+		Kind      string      `yaml:"kind"`
+		Name      string      `yaml:"name"`
+		DataPath  string      `yaml:"data-path,omitempty"`
+		RawConfig interface{} `yaml:"config"`
+	}{}
+	err := unmarshal(&aux)
+	if err != nil {
+		return err
+	}
+
+	dg.Kind = aux.Kind
+	dg.Name = aux.Name
+	dg.DataPath = aux.DataPath
+
+	var cfg datagatherer.Config
+
+	switch dg.Kind {
+	case "gke":
+		cfg = &gke.Config{}
+	case "eks":
+		cfg = &eks.Config{}
+	case "aks":
+		cfg = &aks.Config{}
+	case "k8s":
+		cfg = &k8s.Config{}
+	case "local":
+		cfg = &local.Config{}
+	// dummy dataGatherer is just used for testing
+	case "dummy":
+		cfg = &dummyConfig{}
+	default:
+		return fmt.Errorf("cannot parse data-gatherer configuration, kind %q is not supported", dg.Kind)
+	}
+
+	// we encode aux.RawConfig, which is just a map of reflect.Values, into yaml and decode it again to the right type.
+	err = reMarshal(aux.RawConfig, cfg)
+	if err != nil {
+		return err
+	}
+
+	dg.Config = cfg
+
+	return nil
 }
 
 // Dump generates a YAML string of the Config object

pkg/agent/config_data_gatherers.go

@@ -17,14 +17,13 @@ func TestValidConfigLoad(t *testing.T) {
       schedule: "* * * * *"
       token: "12345"
       data-gatherers:
-      - name: my-gke-cluster
-        kind: gke
-      - name: my-pods
-        kind: k8s
+      - name: d1
+        kind: dummy
+        config:
+          param-1: "bar"
 `
 
 	loadedConfig, err := ParseConfig([]byte(configFileContents))
-
 	if err != nil {
 		t.Errorf("unexpected error: %v", err)
 	}
@@ -39,12 +38,11 @@ func TestValidConfigLoad(t *testing.T) {
 		Token:    "12345",
 		DataGatherers: []dataGatherer{
 			dataGatherer{
-				Name: "my-gke-cluster",
-				Kind: "gke",
-			},
-			dataGatherer{
-				Name: "my-pods",
-				Kind: "k8s",
+				Name: "d1",
+				Kind: "dummy",
+				Config: &dummyConfig{
+					Param1: "bar",
+				},
 			},
 		},
 	}
@@ -99,15 +97,14 @@ func TestPartialMissingConfigError(t *testing.T) {
       schedule: "* * * * *"
       token: "12345"
       data-gatherers:
-        - foo: bar`))
+        - kind: dummy`))
 
 	if parseError == nil {
 		t.Fatalf("expected error, got nil")
 	}
 
 	expectedErrorLines := []string{
-		"2 errors occurred:",
-		"\t* datagatherer 1/1 is missing a kind",
+		"1 error occurred:",
 		"\t* datagatherer 1/1 is missing a name",
 		"\n",
 	}
@@ -120,3 +117,22 @@ func TestPartialMissingConfigError(t *testing.T) {
 		t.Errorf("\ngot=\n%v\nwant=\n%s\ndiff=\n%s", gotError, expectedError, diff.Diff(gotError, expectedError))
 	}
 }
+
+func TestInvalidDataGathered(t *testing.T) {
+	_, parseError := ParseConfig([]byte(`
+      endpoint:
+        host: example.com
+        path: /api/v1/data
+      schedule: "* * * * *"
+      token: "12345"
+      data-gatherers:
+        - kind: "foo"`))
+
+	if parseError == nil {
+		t.Fatalf("expected error, got nil")
+	}
+
+	if got, want := parseError.Error(), `cannot parse data-gatherer configuration, kind "foo" is not supported`; got != want {
+		t.Errorf("\ngot=\n%v\nwant=\n%s\ndiff=\n%s", got, want, diff.Diff(got, want))
+	}
+}

pkg/agent/config_data_gatherers_test.go

@@ -58,39 +58,36 @@ func Run(cmd *cobra.Command, args []string) {
 			log.Printf("Running data gatherer %s of type %s as Local, data-path override present", dgConfig.Name, dgConfig.Kind)
 		}
 
-		dg, err := LoadDataGatherer(ctx, kind, dgConfig.Config)
+		dg, err := dgConfig.Config.NewDataGatherer(ctx)
 		if err != nil {
-			log.Fatalf("Failed to load data gatherer: %s", err)
+			log.Fatalf("failed to instantiate %s DataGatherer: %v", kind, err)
 		}
+
 		dataGatherers[dgConfig.Name] = dg
 	}
 
 	// Fetch from all datagatherers
-	var information []interface{}
+	now := time.Now()
+	readings := []*api.DataReading{}
 	for k, dg := range dataGatherers {
 		i, err := dg.Fetch()
 		if err != nil {
 			log.Fatalf("Error fetching with DataGatherer %q: %s", k, err)
 		}
-		information = append(information, i)
-	}
 
-	fmt.Printf("Gathered Data:\n%+v\n", information)
+		log.Printf("Gathered data for %q:\n", k)
+
+		readings = append(readings, &api.DataReading{
+			DataGatherer: k,
+			Timestamp:    api.Time{Time: now},
+			Data:         i,
+		})
+	}
 
 	for {
 		log.Println("Running Agent...")
 		log.Println("Posting data to ", serverURL)
-		err = postData(serverURL, config.Token, []*api.DataReading{
-			&api.DataReading{
-				DataGatherer: "dummy",
-				Timestamp:    api.Time{Time: time.Now()},
-				Data: map[string]string{
-					"field1": "data1",
-					"field2": "data2",
-					"field3": "data3",
-				},
-			},
-		})
+		err = postData(serverURL, config.Token, readings)
 		// TODO: handle errors gracefully: e.g. handle retries when it is possible
 		if err != nil {
 			log.Fatalf("Post to server failed: %+v", err)