Merge pull request #5 from rhd-gitops-example/extract-more-images

Extract images from more resources.

* Deployments
* StatefulSets
* DaemonSets
* Jobs
* CronJob
* DeploymentConfig

It also simplifies the approach to extracting the data from the resources.

Author: bigkevmcd

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/mitchellh/mapstructure v1.3.2 // indirect
 	github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 // indirect
 	github.com/onsi/ginkgo v1.12.3 // indirect
+	github.com/openshift/api v3.9.0+incompatible
 	github.com/pelletier/go-toml v1.6.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.1.0

go.sum

@@ -202,8 +202,6 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
-github.com/jenkins-x/go-scm v1.5.142 h1:C3Mu/Heb715Bbptelq7yjx8Ux51Q4slxjLLydE5ogkI=
-github.com/jenkins-x/go-scm v1.5.142/go.mod h1:PCT338UhP/pQ0IeEeMEf/hoLTYKcH7qjGEKd7jPkeYg=
 github.com/jenkins-x/go-scm v1.5.151 h1:PrSqpWiRftYXiUniE5XyI1E+4WJMNlyR083q9mBNVTI=
 github.com/jenkins-x/go-scm v1.5.151/go.mod h1:PCT338UhP/pQ0IeEeMEf/hoLTYKcH7qjGEKd7jPkeYg=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
@@ -285,6 +283,8 @@ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/openshift/api v3.9.0+incompatible h1:fJ/KsefYuZAjmrr3+5U9yZIZbTOpVkDDLDLFresAeYs=
+github.com/openshift/api v3.9.0+incompatible/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4=

internal/sets/strings.go

@@ -0,0 +1,27 @@
+package sets
+
+import "sort"
+
+// StringSet is a set of strings with simple functionality for getting a sorted
+// set of elements.
+type StringSet map[string]bool
+
+// NewStringSet creates a new empty StringSet.
+func NewStringSet() StringSet {
+	return StringSet{}
+}
+
+// Add adds a string to the set.
+func (s StringSet) Add(n string) {
+	s[n] = true
+}
+
+// Elements returns a sorted list of elements in the set.
+func (s StringSet) Elements() []string {
+	e := []string{}
+	for k := range s {
+		e = append(e, k)
+	}
+	sort.Strings(e)
+	return e
+}

internal/sets/strings_test.go

@@ -0,0 +1,22 @@
+package sets
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestStringSet(t *testing.T) {
+	s := NewStringSet()
+
+	s.Add("test2")
+	s.Add("test3")
+	s.Add("test1")
+	s.Add("test1")
+
+	want := []string{"test1", "test2", "test3"}
+
+	if diff := cmp.Diff(want, s.Elements()); diff != "" {
+		t.Fatalf("StringSet failed:\n%s", diff)
+	}
+}

pkg/gitfs/gitfs_test.go

@@ -4,9 +4,10 @@ import (
 	"io/ioutil"
 	"testing"
 
-	"github.com/go-git/go-git/v5"
 	"github.com/google/go-cmp/cmp"
 	"sigs.k8s.io/kustomize/pkg/fs"
+
+	"github.com/rhd-gitops-example/gitops-backend/test"
 )
 
 var _ fs.FileSystem = gitFS{}
@@ -105,9 +106,7 @@ func TestNewInMemoryFromOptions(t *testing.T) {
 func makeClonedGFS(t *testing.T) fs.FileSystem {
 	t.Helper()
 	gfs, err := NewInMemoryFromOptions(
-		&git.CloneOptions{
-			URL: "../../",
-		})
+		test.MakeCloneOptions())
 	assertNoError(t, err)
 	return gfs
 }

pkg/httpapi/api_test.go

@@ -16,7 +16,6 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/rhd-gitops-example/gitops-backend/pkg/git"
 	"github.com/rhd-gitops-example/gitops-backend/pkg/parser"
-	"github.com/rhd-gitops-example/gitops-backend/pkg/resource"
 	"github.com/rhd-gitops-example/gitops-backend/test"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/yaml"
@@ -160,7 +159,7 @@ func TestGetPipelinesWithUnknownSecret(t *testing.T) {
 }
 
 func TestGetPipelineApplication(t *testing.T) {
-	testResource := &resource.Resource{
+	testResource := &parser.Resource{
 		Group:     "",
 		Version:   "v1",
 		Kind:      "Deployment",
@@ -211,7 +210,7 @@ func TestGetPipelineApplication(t *testing.T) {
 }
 
 func TestGetPipelineApplicationWithRef(t *testing.T) {
-	testResource := &resource.Resource{
+	testResource := &parser.Resource{
 		Group:     "",
 		Version:   "v1",
 		Kind:      "Deployment",
@@ -426,8 +425,8 @@ func (s stubClientFactory) Create(url, token string) (git.SCM, error) {
 	return s.client, nil
 }
 
-func stubResourceParser(r ...*resource.Resource) parser.ResourceParser {
-	return func(path string, opts *gogit.CloneOptions) ([]*resource.Resource, error) {
+func stubResourceParser(r ...*parser.Resource) parser.ResourceParser {
+	return func(path string, opts *gogit.CloneOptions) ([]*parser.Resource, error) {
 		return r, nil
 	}
 }

pkg/httpapi/application.go

@@ -8,9 +8,11 @@ import (
 
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing/transport/http"
-	"github.com/rhd-gitops-example/gitops-backend/pkg/resource"
+	"github.com/rhd-gitops-example/gitops-backend/pkg/parser"
 )
 
+const nameLabel = "app.kubernetes.io/name"
+
 // TODO: if the environment doesn't exist, this should return a not found error.
 func (a *APIRouter) environmentApplication(authToken string, c *config, envName, appName string) (map[string]interface{}, error) {
 	if c.GitOpsURL == "" {
@@ -47,9 +49,9 @@ func pathForApplication(appName, envName string) string {
 	return path.Join("environments", envName, "apps", appName)
 }
 
-func parseServicesFromResources(env *environment, res []*resource.Resource) ([]responseService, error) {
+func parseServicesFromResources(env *environment, res []*parser.Resource) ([]responseService, error) {
 	serviceImages := map[string]map[string]bool{}
-	serviceResources := map[string][]*resource.Resource{}
+	serviceResources := map[string][]*parser.Resource{}
 	for _, v := range res {
 		name := serviceFromLabels(v.Labels)
 		images, ok := serviceImages[name]
@@ -62,7 +64,7 @@ func parseServicesFromResources(env *environment, res []*resource.Resource) ([]r
 		serviceImages[name] = images
 		resources, ok := serviceResources[name]
 		if !ok {
-			resources = []*resource.Resource{}
+			resources = []*parser.Resource{}
 		}
 		resources = append(resources, v)
 		serviceResources[name] = resources
@@ -92,17 +94,15 @@ func parseServicesFromResources(env *environment, res []*resource.Resource) ([]r
 	return services, nil
 }
 
-const nameLabel = "app.kubernetes.io/name"
-
 func serviceFromLabels(l map[string]string) string {
 	return l[nameLabel]
 }
 
 type responseService struct {
-	Name      string               `json:"name"`
-	Source    source               `json:"source,omitempty"`
-	Images    []string             `json:"images,omitempty"`
-	Resources []*resource.Resource `json:"resources,omitempty"`
+	Name      string             `json:"name"`
+	Source    source             `json:"source,omitempty"`
+	Images    []string           `json:"images,omitempty"`
+	Resources []*parser.Resource `json:"resources,omitempty"`
 }
 
 type source struct {

pkg/httpapi/application_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/rhd-gitops-example/gitops-backend/pkg/resource"
+	"github.com/rhd-gitops-example/gitops-backend/pkg/parser"
 )
 
 const (
@@ -15,7 +15,7 @@ const (
 const testSourceURL = "https://github.com/rhd-example-gitops/gitops-demo.git"
 
 func TestParseServicesFromResources(t *testing.T) {
-	goDemoResources := []*resource.Resource{
+	goDemoResources := []*parser.Resource{
 		{
 			Group: "apps", Version: "v1", Kind: "Deployment", Name: "go-demo-http",
 			Labels: map[string]string{
@@ -40,7 +40,7 @@ func TestParseServicesFromResources(t *testing.T) {
 		},
 	}
 
-	redisResources := []*resource.Resource{
+	redisResources := []*parser.Resource{
 		{
 			Version: "v1", Kind: "Service", Name: "redis",
 			Labels: map[string]string{
@@ -109,7 +109,7 @@ func TestParseServicesFromResources(t *testing.T) {
 }
 
 func TestParseServicesFromResourcesReturnsSetOfImages(t *testing.T) {
-	res := []*resource.Resource{
+	res := []*parser.Resource{
 		{
 			Group: "apps", Version: "v1", Kind: "Deployment", Name: "go-demo-http",
 			Labels: map[string]string{

pkg/parser/converter.go

@@ -0,0 +1,49 @@
+package parser
+
+import (
+	ocpappsv1 "github.com/openshift/api/apps/v1"
+	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
+	batchv1beta1 "k8s.io/api/batch/v1beta1"
+	corev1 "k8s.io/api/core/v1"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+// unstructuredConverter handles conversions between unstructured.Unstructured
+// and some core Kubernetes resource types.
+type unstructuredConverter struct {
+	scheme *runtime.Scheme
+}
+
+// newUnstructuredConverter creates and returns a new UnstructuredConverter.
+func newUnstructuredConverter() (*unstructuredConverter, error) {
+	schemeBuilder := runtime.SchemeBuilder{
+		corev1.AddToScheme,
+		appsv1.AddToScheme,
+		batchv1.AddToScheme,
+		batchv1beta1.AddToScheme,
+		ocpappsv1.AddToScheme,
+	}
+
+	uc := &unstructuredConverter{
+		scheme: runtime.NewScheme(),
+	}
+
+	if err := schemeBuilder.AddToScheme(uc.scheme); err != nil {
+		return nil, err
+	}
+	return uc, nil
+}
+
+// fromUnstructured converts an unstructured.Unstructured to typed struct.
+//
+// If unable to convert using the Kind of obj, then an error is returned.
+func (c *unstructuredConverter) fromUnstructured(o *unstructured.Unstructured) (interface{}, error) {
+	newObj, err := c.scheme.New(o.GetObjectKind().GroupVersionKind())
+	if err != nil {
+		return nil, err
+	}
+	return newObj, c.scheme.Convert(o, newObj, nil)
+}

pkg/parser/extracts.go

@@ -0,0 +1,41 @@
+package parser
+
+import (
+	ocpappsv1 "github.com/openshift/api/apps/v1"
+	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
+	batchv1beta1 "k8s.io/api/batch/v1beta1"
+	corev1 "k8s.io/api/core/v1"
+
+	"github.com/rhd-gitops-example/gitops-backend/internal/sets"
+)
+
+// Deployments, DeploymentConfigs, StatefulSets, DaemonSets, Jobs, CronJobs
+func extractImages(v interface{}) []string {
+	switch k := v.(type) {
+	case *appsv1.Deployment:
+		return extractImagesFromPodTemplateSpec(k.Spec.Template)
+	case *appsv1.StatefulSet:
+		return extractImagesFromPodTemplateSpec(k.Spec.Template)
+	case *appsv1.DaemonSet:
+		return extractImagesFromPodTemplateSpec(k.Spec.Template)
+	case *batchv1.Job:
+		return extractImagesFromPodTemplateSpec(k.Spec.Template)
+	case *batchv1beta1.CronJob:
+		return extractImagesFromPodTemplateSpec(k.Spec.JobTemplate.Spec.Template)
+	case *ocpappsv1.DeploymentConfig:
+		return extractImagesFromPodTemplateSpec(*k.Spec.Template)
+	}
+	return nil
+}
+
+func extractImagesFromPodTemplateSpec(p corev1.PodTemplateSpec) []string {
+	images := sets.NewStringSet()
+	for _, c := range p.Spec.InitContainers {
+		images.Add(c.Image)
+	}
+	for _, c := range p.Spec.Containers {
+		images.Add(c.Image)
+	}
+	return images.Elements()
+}