Merge pull request #210 from jetstack/update-version-checker

Update version checker data gatherer

Author: tfadeyi

go.mod

@@ -19,7 +19,7 @@ require (
 	github.com/googleapis/gnostic v0.5.1 // indirect
 	github.com/hashicorp/go-multierror v1.0.0
 	github.com/imdario/mergo v0.3.11 // indirect
-	github.com/jetstack/version-checker v0.2.1
+	github.com/jetstack/version-checker v0.2.2-0.20201118163251-4bab9ef088ef
 	github.com/juju/errors v0.0.0-20190930114154-d42613fe1ab9
 	github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect
 	github.com/juju/testing v0.0.0-20191001232224-ce9dec17d28b // indirect

go.sum

@@ -295,6 +295,8 @@ github.com/jarcoal/httpmock v1.0.1 h1:OXIOrglWeSllwHQGJ5X4PX4hFZK1DPCXSJVhMSJacg
 github.com/jarcoal/httpmock v1.0.1/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
 github.com/jetstack/version-checker v0.2.1 h1:fTkj1ztb6WP4gMQTROHYAUHJMFMuZOBRRLMrZa0bed4=
 github.com/jetstack/version-checker v0.2.1/go.mod h1:908YLy8SWIJMJoYQpZf2QIRAvBvZ2WM0ylNiCrK8Vvg=
+github.com/jetstack/version-checker v0.2.2-0.20201118163251-4bab9ef088ef h1:bsZIh0Y7TogJfmseOARM9OeZzctSDuuHuJQTQhl97Mo=
+github.com/jetstack/version-checker v0.2.2-0.20201118163251-4bab9ef088ef/go.mod h1:908YLy8SWIJMJoYQpZf2QIRAvBvZ2WM0ylNiCrK8Vvg=
 github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=

pkg/datagatherer/versionchecker/fixtures/manifest-100.json

@@ -1,5 +1,7 @@
 {
   "schemaVersion": 1,
   "name": "jetstack/example",
-  "tag": "v1.0.0"
+  "tag": "v1.0.0",
+  "architecture":"amd64",
+  "os":""
 }

pkg/datagatherer/versionchecker/fixtures/manifest-101.json

@@ -1,5 +1,7 @@
 {
   "schemaVersion": 1,
   "name": "jetstack/example",
-  "tag": "v1.0.1"
+  "tag": "v1.0.1",
+  "architecture":"amd64",
+  "os":""
 }

pkg/datagatherer/versionchecker/fixtures/nodes.json

@@ -0,0 +1,30 @@
+{
+    "apiVersion": "v1",
+    "kind": "List",
+    "metadata": {
+      "resourceVersion": "",
+      "selfLink": ""
+    },
+    "items": [
+        {
+            "apiVersion": "v1",
+            "kind": "Node",
+            "metadata": {
+                "creationTimestamp": "2020-11-24T16:04:38Z",
+                "labels": {
+                    "beta.kubernetes.io/arch": "amd64",
+                    "beta.kubernetes.io/os": "",
+                    "kubernetes.io/arch": "amd64",
+                    "kubernetes.io/hostname": "control-plane",
+                    "kubernetes.io/os": ""
+                },
+                "name": "control-plane",
+                "resourceVersion": "167265",
+                "selfLink": "/api/v1/nodes/control-plane",
+                "uid": "d966826d-e63c-487c-a769-cb02098b95fd"
+            },
+            "spec": {},
+            "status": {}
+        }
+    ]
+}

pkg/datagatherer/versionchecker/fixtures/pods.json.tmpl

@@ -31,6 +31,7 @@
         "uid": "efff9dae-28ca-42c3-be70-970731c44f67"
       },
       "spec": {
+        "nodeName": "control-plane",
         "containers": [
           {
             "name": "example",

pkg/datagatherer/versionchecker/versionchecker.go

@@ -10,11 +10,12 @@ import (
 	"time"
 
 	vcapi "github.com/jetstack/version-checker/pkg/api"
+	vcchecker "github.com/jetstack/version-checker/pkg/checker"
+	vcarchitecture "github.com/jetstack/version-checker/pkg/checker/architecture"
+	vcsearch "github.com/jetstack/version-checker/pkg/checker/search"
+	vcversion "github.com/jetstack/version-checker/pkg/checker/version"
 	vcclient "github.com/jetstack/version-checker/pkg/client"
 	vcselfhosted "github.com/jetstack/version-checker/pkg/client/selfhosted"
-	vcchecker "github.com/jetstack/version-checker/pkg/controller/checker"
-	vcsearch "github.com/jetstack/version-checker/pkg/controller/search"
-	vcversion "github.com/jetstack/version-checker/pkg/version"
 	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -53,7 +54,9 @@ const (
 type Config struct {
 	// the version checker dg will also gather pods and so has the same options
 	// as the dynamic datagatherer
-	Dynamic                     k8s.ConfigDynamic
+	DynamicPod k8s.ConfigDynamic
+	// the nodes information is also gathered by the version checker datagatherer
+	DynamicNode                 k8s.ConfigDynamic
 	VersionCheckerClientOptions vcclient.Options
 	// Currently unused, but keeping to allow future config of VersionChecker
 	VersionCheckerCheckerOptions vcapi.Options
@@ -77,13 +80,20 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		return fmt.Errorf("failed to unmarshal version checker config: %s", err)
 	}
 
-	c.Dynamic.KubeConfigPath = aux.Dynamic.KubeConfigPath
-	c.Dynamic.ExcludeNamespaces = aux.Dynamic.ExcludeNamespaces
-	c.Dynamic.IncludeNamespaces = aux.Dynamic.IncludeNamespaces
+	c.DynamicPod.KubeConfigPath = aux.Dynamic.KubeConfigPath
+	c.DynamicPod.ExcludeNamespaces = aux.Dynamic.ExcludeNamespaces
+	c.DynamicPod.IncludeNamespaces = aux.Dynamic.IncludeNamespaces
 	// gvr must be pods for the version checker dg
-	c.Dynamic.GroupVersionResource.Group = ""
-	c.Dynamic.GroupVersionResource.Version = "v1"
-	c.Dynamic.GroupVersionResource.Resource = "pods"
+	c.DynamicPod.GroupVersionResource.Group = ""
+	c.DynamicPod.GroupVersionResource.Version = "v1"
+	c.DynamicPod.GroupVersionResource.Resource = "pods"
+	// node dynamic dg
+	c.DynamicNode.KubeConfigPath = aux.Dynamic.KubeConfigPath
+	c.DynamicNode.ExcludeNamespaces = []string{}
+	c.DynamicNode.IncludeNamespaces = []string{}
+	c.DynamicNode.GroupVersionResource.Group = ""
+	c.DynamicNode.GroupVersionResource.Version = "v1"
+	c.DynamicNode.GroupVersionResource.Resource = "nodes"
 
 	c.VersionCheckerClientOptions.Selfhosted = map[string]*vcselfhosted.Options{}
 	registryKindCounts := map[string]int{}
@@ -200,7 +210,12 @@ func loadKeysFromPaths(keys []string, params map[string]string) (map[string]stri
 // NewDataGatherer creates a new VersionChecker DataGatherer
 func (c *Config) NewDataGatherer(ctx context.Context) (datagatherer.DataGatherer, error) {
 	// create the k8s DataGatherer to use when collecting pods
-	dynamicDg, err := c.Dynamic.NewDataGatherer(ctx)
+	podDynamicDg, err := c.DynamicPod.NewDataGatherer(ctx)
+	if err != nil {
+		return nil, err
+	}
+	// create a data gatherer to use to collect nodes architecture information
+	nodeDynamicDg, err := c.DynamicNode.NewDataGatherer(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -219,13 +234,16 @@ func (c *Config) NewDataGatherer(ctx context.Context) (datagatherer.DataGatherer
 		timeout,
 		vcversion.New(log, imageClient, timeout),
 	)
+	architecture := vcarchitecture.New()
 
 	// dg wraps version checker and dynamic client to request pods
 	return &DataGatherer{
 		ctx:                   ctx,
 		config:                c,
-		dynamicDg:             dynamicDg,
-		versionChecker:        vcchecker.New(search),
+		podDynamicDg:          podDynamicDg,
+		nodeDynamicDg:         nodeDynamicDg,
+		nodeArchitecture:      architecture,
+		versionChecker:        vcchecker.New(search, architecture),
 		versionCheckerLog:     log,
 		versionCheckerOptions: c.VersionCheckerCheckerOptions,
 	}, nil
@@ -235,7 +253,9 @@ func (c *Config) NewDataGatherer(ctx context.Context) (datagatherer.DataGatherer
 type DataGatherer struct {
 	ctx                   context.Context
 	config                *Config
-	dynamicDg             datagatherer.DataGatherer
+	podDynamicDg          datagatherer.DataGatherer
+	nodeDynamicDg         datagatherer.DataGatherer
+	nodeArchitecture      *vcarchitecture.NodeMap
 	versionChecker        *vcchecker.Checker
 	versionCheckerLog     *logrus.Entry
 	versionCheckerOptions vcapi.Options
@@ -257,9 +277,31 @@ type containerResult struct {
 
 // Fetch retrieves cluster information from GKE.
 func (g *DataGatherer) Fetch() (interface{}, error) {
-	rawPods, err := g.dynamicDg.Fetch()
+	// Get nodes information to update version-checker architecture structure
+	rawNodes, err := g.nodeDynamicDg.Fetch()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to fetch nodes: %v", err)
+	}
+	nodes, ok := rawNodes.(*unstructured.UnstructuredList)
+	if !ok {
+		return nil, fmt.Errorf("failed to parse nodes loaded from DataGatherer")
+	}
+	for _, v := range nodes.Items {
+		var node v1.Node
+		err := runtime.DefaultUnstructuredConverter.FromUnstructured(v.Object, &node)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse node from unstructured data: %v", err)
+		}
+		// update version-checker's internal representation of the current cluster's nodes,
+		// to correctly select the right OS and Architecture for the images
+		if err = g.nodeArchitecture.Add(&node); err != nil {
+			return nil, fmt.Errorf("failed to add node to version-checker architecture structure")
+		}
+	}
+
+	rawPods, err := g.podDynamicDg.Fetch()
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch pods: %v", err)
 	}
 
 	pods, ok := rawPods.(*unstructured.UnstructuredList)
@@ -272,7 +314,7 @@ func (g *DataGatherer) Fetch() (interface{}, error) {
 		var pod v1.Pod
 		err = runtime.DefaultUnstructuredConverter.FromUnstructured(v.Object, &pod)
 		if err != nil {
-			return nil, fmt.Errorf("failed to parse pod from unstructured data")
+			return nil, fmt.Errorf("failed to parse pod from unstructured data: %v", err)
 		}
 
 		var allContainers []v1.Container

pkg/datagatherer/versionchecker/versionchecker_test.go

@@ -12,6 +12,7 @@ import (
 	"net/url"
 	"os"
 	"reflect"
+	"regexp"
 	"testing"
 
 	"gopkg.in/yaml.v2"
@@ -78,23 +79,23 @@ registries:
 		expTarget    interface{}
 	}{
 		"KubeConfigPath": {
-			configTarget: cfg.Dynamic.KubeConfigPath,
+			configTarget: cfg.DynamicPod.KubeConfigPath,
 			expTarget:    "/home/someone/.kube/config",
 		},
 		"GVR": {
-			configTarget: cfg.Dynamic.GroupVersionResource,
+			configTarget: cfg.DynamicPod.GroupVersionResource,
 			expTarget: schema.GroupVersionResource{
 				Group:    "",
 				Version:  "v1",
 				Resource: "pods", // should use pods even if other gvr set
 			},
 		},
 		"IncludeNamespaces": {
-			configTarget: cfg.Dynamic.IncludeNamespaces,
+			configTarget: cfg.DynamicPod.IncludeNamespaces,
 			expTarget:    []string{"default"},
 		},
 		"ExcludeNamespaces": {
-			configTarget: cfg.Dynamic.ExcludeNamespaces,
+			configTarget: cfg.DynamicPod.ExcludeNamespaces,
 			expTarget:    []string{"kube-system"},
 		},
 		"GCR Token": {
@@ -173,12 +174,12 @@ registries:
 
 	rawResults, err := dg.Fetch()
 	if err != nil {
-		t.Fatalf("failed fetch data %s", err)
+		t.Fatalf("failed fetch data: %s", err)
 	}
 
 	resultsJSON, err := json.MarshalIndent(rawResults, "", "  ")
 	if err != nil {
-		t.Fatalf("failed to marshal data %s", err)
+		t.Fatalf("failed to marshal data: %s", err)
 	}
 
 	expectedResultsJSON := fmt.Sprintf(`[
@@ -218,7 +219,8 @@ registries:
             ],
             "resources": {}
           }
-        ]
+        ],
+        "nodeName": "control-plane"
       },
       "status": {
         "containerStatuses": [
@@ -241,16 +243,20 @@ registries:
         "result": {
           "CurrentVersion": "v1.0.0",
           "LatestVersion": "v1.0.1",
-          "IsLatest": false,
-          "ImageURL": "%s/jetstack/example"
+	  "IsLatest": false,
+	  "ImageURL": "%s/jetstack/example",
+	  "OS": "",
+	  "Architecture": "amd64"
         }
       }
     ]
   }
 ]`, parsedURL.Host, parsedURL.Host)
 
-	if expectedResultsJSON != string(resultsJSON) {
-		t.Fatalf("results json does not match: %s vs %s", resultsJSON, expectedResultsJSON)
+	act := removeWhitespace(string(resultsJSON))
+	expected := removeWhitespace(expectedResultsJSON)
+	if act != expected {
+		t.Fatalf("results json does not match: %s vs %s", act, expected)
 	}
 }
 
@@ -319,6 +325,11 @@ func createLocalTestServer(t *testing.T) *httptest.Server {
 		var err error
 
 		switch r.URL.Path {
+		case "/api/v1/nodes":
+			responseContent, err = ioutil.ReadFile("fixtures/nodes.json")
+			if err != nil {
+				t.Fatalf("failed to read nodes fixture: %s", err)
+			}
 		case "/api/v1/pods":
 			// the responses from the server are self referential and the host is
 			// needed to generate responses
@@ -347,17 +358,23 @@ func createLocalTestServer(t *testing.T) *httptest.Server {
 			}
 		case "/v2/jetstack/example/manifests/v1.0.0":
 			// this is a partial response, but it's all version checker needs
+			// selfhosted registers currently don't expose OS information
 			responseContent = []byte(`{
 			  "schemaVersion": 1,
 			  "name": "jetstack/example",
-			  "tag": "v1.0.0"
+			  "tag": "v1.0.0",
+			  "architecture":"amd64",
+			  "os":""
 			}`)
 		case "/v2/jetstack/example/manifests/v1.0.1":
 			// this is a partial response, but it's all version checker needs
+			// selfhosted registers currently don't expose OS information
 			responseContent = []byte(`{
 			  "schemaVersion": 1,
 			  "name": "jetstack/example",
-			  "tag": "v1.0.1"
+			  "tag": "v1.0.1",
+			  "architecture":"amd64",
+			  "os":""
 			}`)
 		default:
 			t.Fatalf("Unexpected URL was called: %s", r.URL.Path)
@@ -368,3 +385,9 @@ func createLocalTestServer(t *testing.T) *httptest.Server {
 
 	return localServer
 }
+
+// Simple helper function to remove whitespaces from string
+func removeWhitespace(input string) string {
+	ws := regexp.MustCompile(`[\s]`)
+	return ws.ReplaceAllString(input, "")
+}