Scrape only Included namespaces when set to allow more restricted RBAC (#156)

* Test empty gvr validation

Signed-off-by: Charlie Egan <[email protected]>

* Add IncludedNamespaces

Signed-off-by: Charlie Egan <[email protected]>

* Support fetching from many namespaces

Signed-off-by: Charlie Egan <[email protected]>

* Fetch mutiple namespaces with many requests

Signed-off-by: Charlie Egan <[email protected]>

* Add test for setting of namespaces from config

Signed-off-by: Charlie Egan <[email protected]>

* Fix typo

Signed-off-by: Charlie Egan <[email protected]>

Author: charlieegan3

pkg/datagatherer/k8s/generic.go

@@ -3,6 +3,7 @@ package k8s
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"github.com/jetstack/preflight/pkg/datagatherer"
 	"github.com/pkg/errors"
@@ -22,6 +23,8 @@ type Config struct {
 	GroupVersionResource schema.GroupVersionResource
 	// ExcludeNamespaces is a list of namespaces to exclude.
 	ExcludeNamespaces []string `yaml:"exclude-namespaces"`
+	// IncludeNamespaces is a list of namespaces to include.
+	IncludeNamespaces []string `yaml:"include-namespaces"`
 }
 
 // UnmarshalYAML unmarshals the Config resolving GroupVersionResource.
@@ -51,8 +54,17 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 // validate validates the configuration.
 func (c *Config) validate() error {
+	var errors []string
+	if len(c.ExcludeNamespaces) > 0 && len(c.IncludeNamespaces) > 0 {
+		errors = append(errors, "cannot set excluded and included namespaces")
+	}
+
 	if c.GroupVersionResource.Resource == "" {
-		return fmt.Errorf("invalid configuration: GroupVersionResource.Resource cannot be empty")
+		errors = append(errors, "invalid configuration: GroupVersionResource.Resource cannot be empty")
+	}
+
+	if len(errors) > 0 {
+		return fmt.Errorf(strings.Join(errors, ", "))
 	}
 
 	return nil
@@ -61,19 +73,24 @@ func (c *Config) validate() error {
 // NewDataGatherer constructs a new instance of the generic K8s data-gatherer for the provided
 // GroupVersionResource.
 func (c *Config) NewDataGatherer(ctx context.Context) (datagatherer.DataGatherer, error) {
-	if err := c.validate(); err != nil {
+	cl, err := NewDynamicClient(c.KubeConfigPath)
+	if err != nil {
 		return nil, err
 	}
 
-	cl, err := NewDynamicClient(c.KubeConfigPath)
-	if err != nil {
+	return c.newDataGathererWithClient(cl)
+}
+
+func (c *Config) newDataGathererWithClient(cl dynamic.Interface) (datagatherer.DataGatherer, error) {
+	if err := c.validate(); err != nil {
 		return nil, err
 	}
 
 	return &DataGatherer{
 		cl:                   cl,
 		groupVersionResource: c.GroupVersionResource,
 		fieldSelector:        generateFieldSelector(c.ExcludeNamespaces),
+		namespaces:           c.IncludeNamespaces,
 	}, nil
 }
 
@@ -92,7 +109,7 @@ type DataGatherer struct {
 	// namespace, if specified, limits the namespace of the resources returned.
 	// This field *must* be omitted when the groupVersionResource refers to a
 	// non-namespaced resource.
-	namespace string
+	namespaces []string
 	// fieldSelector is a field selector string used to filter resources
 	// returned by the Kubernetes API.
 	// https://kubernetes.io/docs/concepts/overview/working-with-objects/field-selectors/
@@ -105,19 +122,34 @@ func (g *DataGatherer) Fetch() (interface{}, error) {
 	if g.groupVersionResource.Resource == "" {
 		return nil, fmt.Errorf("resource type must be specified")
 	}
-	resourceInterface := namespaceResourceInterface(g.cl.Resource(g.groupVersionResource), g.namespace)
-	list, err := resourceInterface.List(metav1.ListOptions{
-		FieldSelector: g.fieldSelector,
-	})
-	if err != nil {
-		return nil, errors.WithStack(err)
+
+	var list unstructured.UnstructuredList
+
+	fetchNamespaces := g.namespaces
+	if len(fetchNamespaces) == 0 {
+		// then they must have been looking for all namespaces
+		fetchNamespaces = []string{""}
 	}
+
+	for _, namespace := range fetchNamespaces {
+		resourceInterface := namespaceResourceInterface(g.cl.Resource(g.groupVersionResource), namespace)
+		namespaceList, err := resourceInterface.List(metav1.ListOptions{
+			FieldSelector: g.fieldSelector,
+		})
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+		list.Object = namespaceList.Object
+		list.Items = append(list.Items, namespaceList.Items...)
+	}
+
 	// Redact Secret data
-	err = redactList(list)
+	err := redactList(&list)
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}
-	return list, nil
+
+	return &list, nil
 }
 
 func redactList(list *unstructured.UnstructuredList) error {

pkg/datagatherer/k8s/generic_test.go

@@ -3,6 +3,7 @@ package k8s
 import (
 	"encoding/json"
 	"reflect"
+	"strings"
 	"testing"
 
 	"gopkg.in/yaml.v2"
@@ -60,14 +61,39 @@ func asUnstructuredList(items ...*unstructured.Unstructured) *unstructured.Unstr
 	}
 }
 
+func TestNewDataGathererWithClient(t *testing.T) {
+	config := Config{
+		IncludeNamespaces:    []string{"a"},
+		GroupVersionResource: schema.GroupVersionResource{Group: "foobar", Version: "v1", Resource: "foos"},
+	}
+	cl := fake.NewSimpleDynamicClient(runtime.NewScheme())
+	dg, err := config.newDataGathererWithClient(cl)
+
+	if err != nil {
+		t.Errorf("expected no error but got: %v", err)
+	}
+
+	expected := &DataGatherer{
+		cl:                   cl,
+		groupVersionResource: config.GroupVersionResource,
+		// it's important that the namespaces are set as the IncludeNamespaces
+		// during initialization
+		namespaces: config.IncludeNamespaces,
+	}
+
+	if !reflect.DeepEqual(dg, expected) {
+		t.Errorf("unexpected difference: %v", diff.ObjectDiff(dg, expected))
+	}
+}
+
 func TestGenericGatherer_Fetch(t *testing.T) {
 	emptyScheme := runtime.NewScheme()
 	tests := map[string]struct {
-		gvr       schema.GroupVersionResource
-		namespace string
-		objects   []runtime.Object
-		expected  interface{}
-		err       bool
+		gvr        schema.GroupVersionResource
+		namespaces []string
+		objects    []runtime.Object
+		expected   interface{}
+		err        bool
 	}{
 		"an error should be returned if 'resource' is missing": {
 			err: true,
@@ -85,8 +111,8 @@ func TestGenericGatherer_Fetch(t *testing.T) {
 			),
 		},
 		"only Foos in the specified namespace should be returned": {
-			gvr:       schema.GroupVersionResource{Group: "foobar", Version: "v1", Resource: "foos"},
-			namespace: "testns",
+			gvr:        schema.GroupVersionResource{Group: "foobar", Version: "v1", Resource: "foos"},
+			namespaces: []string{"testns"},
 			objects: []runtime.Object{
 				getObject("foobar/v1", "Foo", "testfoo", "testns"),
 				getObject("foobar/v1", "Foo", "testfoo", "nottestns"),
@@ -121,6 +147,19 @@ func TestGenericGatherer_Fetch(t *testing.T) {
 				getSecret("anothertestsecret", "differentns", map[string]interface{}{}, false),
 			),
 		},
+		"Foos in different namespaces should be returned if they are in the namespace list for the gatherer": {
+			gvr:        schema.GroupVersionResource{Group: "foobar", Version: "v1", Resource: "foos"},
+			namespaces: []string{"testns", "testns2"},
+			objects: []runtime.Object{
+				getObject("foobar/v1", "Foo", "testfoo", "testns"),
+				getObject("foobar/v1", "Foo", "testfoo2", "testns2"),
+				getObject("foobar/v1", "Foo", "testfoo3", "nottestns"),
+			},
+			expected: asUnstructuredList(
+				getObject("foobar/v1", "Foo", "testfoo", "testns"),
+				getObject("foobar/v1", "Foo", "testfoo2", "testns2"),
+			),
+		},
 		// Note that we can't test use of fieldSelector to exclude namespaces
 		// here as the as the fake client does not implement it.
 		// See go/pkg/mod/k8s.io/[email protected]/dynamic/fake/simple.go:291
@@ -132,7 +171,9 @@ func TestGenericGatherer_Fetch(t *testing.T) {
 			g := DataGatherer{
 				cl:                   cl,
 				groupVersionResource: test.gvr,
-				namespace:            test.namespace,
+				// if empty, namespaces will default to []string{""} during
+				// fetch to get all ns
+				namespaces: test.namespaces,
 			}
 
 			res, err := g.Fetch()
@@ -191,6 +232,38 @@ exclude-namespaces:
 	}
 }
 
+func TestConfigValidate(t *testing.T) {
+	tests := []struct {
+		Config        Config
+		ExpectedError string
+	}{
+		{
+			Config: Config{
+				GroupVersionResource: schema.GroupVersionResource{
+					Group:    "",
+					Version:  "",
+					Resource: "",
+				},
+			},
+			ExpectedError: "invalid configuration: GroupVersionResource.Resource cannot be empty",
+		},
+		{
+			Config: Config{
+				IncludeNamespaces: []string{"a"},
+				ExcludeNamespaces: []string{"b"},
+			},
+			ExpectedError: "cannot set excluded and included namespaces",
+		},
+	}
+
+	for _, test := range tests {
+		err := test.Config.validate()
+		if !strings.Contains(err.Error(), test.ExpectedError) {
+			t.Errorf("expected %s, got %s", test.ExpectedError, err.Error())
+		}
+	}
+}
+
 func TestGenerateFieldSelector(t *testing.T) {
 	tests := []struct {
 		ExcludeNamespaces     []string