feat: added api to get all applications using the Application CRs (#20)

* feat: get all applications using the Application CR

* added changes to verify gitops url

* fixed issues

* added unit tests

Author: shubhamagarwal19

go.mod

@@ -3,48 +3,62 @@ module github.com/redhat-developer/gitops-backend
 go 1.15
 
 require (
+	github.com/argoproj/argo-cd v0.8.1-0.20210326223336-719d6a9c252e
+	github.com/argoproj/pkg v0.9.0 // indirect
 	github.com/emicklei/go-restful v2.12.0+incompatible // indirect
-	github.com/evanphx/json-patch v4.5.0+incompatible // indirect
 	github.com/go-git/go-git/v5 v5.1.0
-	github.com/go-openapi/jsonreference v0.19.3 // indirect
 	github.com/go-openapi/spec v0.19.8 // indirect
 	github.com/go-openapi/swag v0.19.9 // indirect
-	github.com/gogo/protobuf v1.3.1 // indirect
-	github.com/google/go-cmp v0.4.1
-	github.com/google/gofuzz v1.1.0 // indirect
-	github.com/googleapis/gnostic v0.4.0 // indirect
+	github.com/google/go-cmp v0.5.2
 	github.com/jenkins-x/go-scm v1.5.151
-	github.com/json-iterator/go v1.1.9 // indirect
 	github.com/julienschmidt/httprouter v1.2.0
-	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
 	github.com/mailru/easyjson v0.7.1 // indirect
 	github.com/mitchellh/mapstructure v1.3.2 // indirect
 	github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 // indirect
-	github.com/onsi/ginkgo v1.12.3 // indirect
 	github.com/openshift/api v3.9.0+incompatible
 	github.com/pelletier/go-toml v1.6.0 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.1.0
-	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect
-	github.com/prometheus/common v0.7.0 // indirect
-	github.com/prometheus/procfs v0.0.5 // indirect
+	github.com/prometheus/client_golang v1.7.1
+	github.com/robfig/cron v1.2.0 // indirect
 	github.com/shurcooL/githubv4 v0.0.0-20191102174205-af46314aec7b // indirect
+	github.com/sirupsen/logrus v1.7.0
 	github.com/spf13/cast v1.3.1 // indirect
-	github.com/spf13/cobra v1.0.0
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/cobra v1.1.1
 	github.com/spf13/viper v1.7.0
-	github.com/stretchr/testify v1.6.1 // indirect
-	go.uber.org/zap v1.10.0
-	golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 // indirect
-	golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect
-	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
-	google.golang.org/appengine v1.6.5 // indirect
-	gopkg.in/ini.v1 v1.52.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
-	k8s.io/api v0.17.6
-	k8s.io/apimachinery v0.17.6
-	k8s.io/client-go v0.17.6
-	k8s.io/utils v0.0.0-20200619165400-6e3d28b6ed19 // indirect
+	go.uber.org/zap v1.15.0
+	k8s.io/api v0.21.0
+	k8s.io/apimachinery v0.21.0
+	k8s.io/client-go v11.0.1-0.20190816222228-6d55c1b1f1ca+incompatible
+	k8s.io/kubernetes v1.21.0 // indirect
+	sigs.k8s.io/controller-runtime v0.8.3
 	sigs.k8s.io/kustomize v2.0.3+incompatible
 	sigs.k8s.io/yaml v1.2.0
 )
+
+replace (
+	k8s.io/api => k8s.io/api v0.21.0
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.21.0
+	k8s.io/apimachinery => k8s.io/apimachinery v0.21.1-rc.0
+	k8s.io/apiserver => k8s.io/apiserver v0.21.0
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.21.0
+	k8s.io/client-go => k8s.io/client-go v0.21.0
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.21.0
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.21.0
+	k8s.io/code-generator => k8s.io/code-generator v0.21.1-rc.0
+	k8s.io/component-base => k8s.io/component-base v0.21.0
+	k8s.io/component-helpers => k8s.io/component-helpers v0.21.0
+	k8s.io/controller-manager => k8s.io/controller-manager v0.21.0
+	k8s.io/cri-api => k8s.io/cri-api v0.21.1-rc.0
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.21.0
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.21.0
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.21.0
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.21.0
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.21.0
+	k8s.io/kubectl => k8s.io/kubectl v0.21.0
+	k8s.io/kubelet => k8s.io/kubelet v0.21.0
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.21.0
+	k8s.io/metrics => k8s.io/metrics v0.21.0
+	k8s.io/mount-utils => k8s.io/mount-utils v0.21.1-rc.0
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.21.0
+	k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.21.0
+	k8s.io/sample-controller => k8s.io/sample-controller v0.21.0
+)

go.sum

@@ -5,11 +5,14 @@ import (
 	"log"
 	"net/http"
 
+	argoV1aplha1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/redhat-developer/gitops-backend/pkg/git"
 	"github.com/redhat-developer/gitops-backend/pkg/health"
 	"github.com/redhat-developer/gitops-backend/pkg/httpapi"
@@ -27,6 +30,9 @@ const (
 
 func init() {
 	cobra.OnInitialize(initConfig)
+	if err := argoV1aplha1.AddToScheme(scheme.Scheme); err != nil {
+		log.Fatalf("failed to initialize ArgoCD scheme, err: %v", err)
+	}
 }
 
 func logIfError(e error) {
@@ -127,6 +133,10 @@ func makeAPIRouter(m metrics.Interface) (*httpapi.APIRouter, error) {
 	secretGetter := secrets.NewFromConfig(
 		&rest.Config{Host: config.Host},
 		viper.GetBool(insecureFlag))
-	router := httpapi.NewRouter(cf, secretGetter)
+	k8sClient, err := ctrlclient.New(config, ctrlclient.Options{})
+	if err != nil {
+		return nil, err
+	}
+	router := httpapi.NewRouter(cf, secretGetter, k8sClient)
 	return router, nil
 }

pkg/cmd/root.go

@@ -9,8 +9,10 @@ import (
 	"net/url"
 	"strings"
 
+	argoV1aplha1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
 	"github.com/julienschmidt/httprouter"
 	"k8s.io/apimachinery/pkg/types"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/yaml"
 
 	"github.com/redhat-developer/gitops-backend/pkg/git"
@@ -33,18 +35,21 @@ type APIRouter struct {
 	secretGetter     secrets.SecretGetter
 	secretRef        types.NamespacedName
 	resourceParser   parser.ResourceParser
+	k8sClient        ctrlclient.Client
 }
 
 // NewRouter creates and returns a new APIRouter.
-func NewRouter(c git.ClientFactory, s secrets.SecretGetter) *APIRouter {
+func NewRouter(c git.ClientFactory, s secrets.SecretGetter, kc ctrlclient.Client) *APIRouter {
 	api := &APIRouter{
 		Router:           httprouter.New(),
 		gitClientFactory: c,
 		secretGetter:     s,
 		secretRef:        DefaultSecretRef,
 		resourceParser:   parser.ParseFromGit,
+		k8sClient:        kc,
 	}
 	api.HandlerFunc(http.MethodGet, "/pipelines", api.GetPipelines)
+	api.HandlerFunc(http.MethodGet, "/applications", api.ListApplications)
 	api.HandlerFunc(http.MethodGet, "/environments/:env/application/:app", api.GetApplication)
 	return api
 }
@@ -161,6 +166,41 @@ func (a *APIRouter) GetApplication(w http.ResponseWriter, r *http.Request) {
 	marshalResponse(w, appEnvironments)
 }
 
+func (a *APIRouter) ListApplications(w http.ResponseWriter, r *http.Request) {
+	repoURL := strings.TrimSpace(r.URL.Query().Get("url"))
+	if repoURL == "" {
+		http.Error(w, "please provide a valid GitOps repo URL", http.StatusBadRequest)
+		return
+	}
+
+	parsedRepoURL, err := url.Parse(repoURL)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to parse URL, error: %v", err), http.StatusBadRequest)
+		return
+	}
+
+	parsedRepoURL.RawQuery = ""
+
+	appList := &argoV1aplha1.ApplicationList{}
+	var listOptions []ctrlclient.ListOption
+
+	listOptions = append(listOptions, ctrlclient.InNamespace(""))
+
+	err = a.k8sClient.List(r.Context(), appList, listOptions...)
+	if err != nil {
+		log.Printf("ERROR: failed to get application list: %v", err)
+		http.Error(w, fmt.Sprintf("failed to get list of application, err: %v", err), http.StatusBadRequest)
+		return
+	}
+
+	apps := make([]*argoV1aplha1.Application, 0)
+	for _, app := range appList.Items {
+		apps = append(apps, app.DeepCopy())
+	}
+
+	marshalResponse(w, applicationsToAppsResponse(apps, parsedRepoURL.String()))
+}
+
 func (a *APIRouter) getAuthToken(ctx context.Context, req *http.Request) (string, error) {
 	token := AuthToken(ctx)
 	secret, ok := secretRefFromQuery(req.URL.Query())

pkg/httpapi/api.go

@@ -12,12 +12,16 @@ import (
 	"strings"
 	"testing"
 
+	argoV1aplha1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
 	gogit "github.com/go-git/go-git/v5"
 	"github.com/google/go-cmp/cmp"
 	"github.com/redhat-developer/gitops-backend/pkg/git"
 	"github.com/redhat-developer/gitops-backend/pkg/parser"
 	"github.com/redhat-developer/gitops-backend/test"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes/scheme"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"sigs.k8s.io/yaml"
 )
 
@@ -290,6 +294,72 @@ func TestParseURL(t *testing.T) {
 	}
 }
 
+func TestListApplications(t *testing.T) {
+	err := argoV1aplha1.AddToScheme(scheme.Scheme)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	builder := fake.NewClientBuilder()
+	kc := builder.Build()
+
+	ts, _ := makeServer(t, func(router *APIRouter) {
+		router.k8sClient = kc
+	})
+
+	var createOptions []ctrlclient.CreateOption
+	app, _ := testArgoApplication()
+	err = kc.Create(context.TODO(), app, createOptions...)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	url := "https://github.com/test-repo/gitops.git?ref=HEAD"
+	req := makeClientRequest(t, "Bearer testing", fmt.Sprintf("%s/applications?url=%s", ts.URL, url))
+	res, err := ts.Client().Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertJSONResponse(t, res, map[string]interface{}{
+		"applications": []interface{}{
+			map[string]interface{}{
+				"name":         "test-app",
+				"repo_url":     "https://github.com/test-repo/gitops.git",
+				"environments": []interface{}{"dev"},
+			},
+		},
+	})
+}
+
+func TestListApplications_badURL(t *testing.T) {
+	builder := fake.NewClientBuilder()
+	kc := builder.Build()
+
+	ts, _ := makeServer(t, func(router *APIRouter) {
+		router.k8sClient = kc
+	})
+
+	req := makeClientRequest(t, "Bearer testing", fmt.Sprintf("%s/applications", ts.URL))
+	resp, err := ts.Client().Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertHTTPError(t, resp, http.StatusBadRequest, "please provide a valid GitOps repo URL")
+}
+
+func testArgoApplication() (*argoV1aplha1.Application, error) {
+	applicationYaml, _ := ioutil.ReadFile("testdata/application.yaml")
+	app := &argoV1aplha1.Application{}
+	err := yaml.Unmarshal(applicationYaml, app)
+	if err != nil {
+		return nil, err
+	}
+
+	return app, err
+}
+
 func newClient() *stubClient {
 	return &stubClient{files: make(map[string]string)}
 }
@@ -347,7 +417,8 @@ func makeServer(t *testing.T, opts ...routerOptionFunc) (*httptest.Server, *stub
 		testKey:       "token",
 	}
 	sf := &stubClientFactory{client: newClient()}
-	router := NewRouter(sf, sg)
+	var kc ctrlclient.Client
+	router := NewRouter(sf, sg, kc)
 	for _, o := range opts {
 		o(router)
 	}

pkg/httpapi/api_test.go

@@ -1,6 +1,12 @@
 package httpapi
 
-import "sort"
+import (
+	log "github.com/sirupsen/logrus"
+	"sort"
+	"strings"
+
+	argoV1aplha1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
+)
 
 // TODO: this should really import the config from the upstream and use it to
 // unmarshal.
@@ -24,3 +30,42 @@ func pipelinesToAppsResponse(cfg *config) *appsResponse {
 	}
 	return &appsResponse{Apps: apps}
 }
+
+func applicationsToAppsResponse(appSet []*argoV1aplha1.Application, repoURL string) *appsResponse {
+	appsMap := make(map[string]appResponse)
+	var appName string
+	repoURL = strings.TrimSuffix(repoURL, ".git")
+
+	for _, app := range appSet {
+		if repoURL != strings.TrimSuffix(app.Spec.Source.RepoURL, ".git") {
+			log.Printf("repoURL[%v], doesn not match with Source Repo URL[%v]", repoURL, strings.TrimSuffix(app.Spec.Source.RepoURL, ".git"))
+			continue
+		}
+		if app.ObjectMeta.Labels != nil {
+			appName = app.ObjectMeta.Labels["app.kubernetes.io/name"]
+		}
+
+		if appName == "" {
+			log.Println("AppName is empty")
+			continue
+		}
+
+		if appResp, ok := appsMap[appName]; !ok {
+			appsMap[appName] = appResponse{
+				Name:         appName,
+				RepoURL:      app.Spec.Source.RepoURL,
+				Environments: []string{app.Spec.Destination.Namespace},
+			}
+		} else {
+			appResp.Environments = append(appResp.Environments, app.Spec.Destination.Namespace)
+			appsMap[appName] = appResp
+		}
+	}
+
+	var apps []appResponse
+	for _, app := range appsMap {
+		apps = append(apps, app)
+	}
+
+	return &appsResponse{Apps: apps}
+}

pkg/httpapi/conversion.go

@@ -1,6 +1,8 @@
 package httpapi
 
 import (
+	"fmt"
+	argoV1aplha1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -23,3 +25,27 @@ func TestPipelinesToAppsResponse(t *testing.T) {
 		t.Fatalf("failed to parse:\n%s", diff)
 	}
 }
+
+func TestApplicationsToAppsResponse(t *testing.T) {
+	var apps []*argoV1aplha1.Application
+	app, _ := testArgoApplication()
+	apps = append(apps, app)
+
+	want := &appsResponse{
+		Apps: []appResponse{
+			{Name: "test-app", RepoURL: "https://github.com/test-repo/gitops.git", Environments: []string{"dev"}},
+		},
+	}
+
+	resp := applicationsToAppsResponse(apps, "https://github.com/test-repo/gitops")
+
+	if diff := cmp.Diff(want, resp); diff != "" {
+		t.Fatal(fmt.Errorf("WANT[%v] != RECEIVED[%v], diff=%s", want, resp, diff))
+	}
+
+	resp = applicationsToAppsResponse(apps, "https://github.com/test-repo/gitops.git")
+
+	if diff := cmp.Diff(want, resp); diff != "" {
+		t.Fatal(fmt.Errorf("WANT[%v] != RECEIVED[%v], diff=%s", want, resp, diff))
+	}
+}