FEATURE: Azure gitops support (#458)

* git azure service added

* wip

* wip

* wip

* azure devops support dev

* git provider type correction

* error handling correction

* error handling for branch  creation

* branch error check fix

* add git binary

* add git binary

* local tested

* fix azure gitops startup

* git clone fix

* log corection

* review fix

* for loop simplification

* db migration added

* gitops config default handling

* slow git clone handling

Co-authored-by: Vikram Singh <[email protected]>

Author: nishant-d

Dockerfile

@@ -11,11 +11,15 @@ RUN GOOS=linux make
 
 FROM alpine:3.9
 RUN apk add --no-cache ca-certificates
-RUN echo pwd
+RUN apk update
+RUN apk add git
 COPY --from=build-env  /go/src/github.com/devtron-labs/devtron/devtron .
 COPY --from=build-env  /go/src/github.com/devtron-labs/devtron/auth_model.conf .
 COPY --from=build-env  /go/src/github.com/devtron-labs/devtron/vendor/github.com/argoproj/argo-cd/assets/ /go/src/github.com/devtron-labs/devtron/vendor/github.com/argoproj/argo-cd/assets
 COPY --from=build-env  /go/src/github.com/devtron-labs/devtron/scripts/devtron-reference-helm-charts scripts/devtron-reference-helm-charts
 COPY --from=build-env  /go/src/github.com/devtron-labs/devtron/scripts/argo-assets/APPLICATION_TEMPLATE.JSON scripts/argo-assets/APPLICATION_TEMPLATE.JSON
 
+COPY ./git-ask-pass.sh /git-ask-pass.sh
+RUN chmod +x /git-ask-pass.sh
+
 CMD ["./devtron"]

Gopkg.lock

@@ -187,3 +187,7 @@ required = [
 [[constraint]]
   name = "github.com/igm/sockjs-go"
   version = "3.0.0"
+
+[[constraint]]
+  branch = "dev"
+  name = "github.com/microsoft/azure-devops-go-api"

Gopkg.toml

@@ -646,6 +646,7 @@ func InitializeApp() (*App, error) {
 		wire.Bind(new(router.CommonRouter), new(*router.CommonRouterImpl)),
 		restHandler.NewCommonRestHanlderImpl,
 		wire.Bind(new(restHandler.CommonRestHanlder), new(*restHandler.CommonRestHanlderImpl)),
+		util.NewGitCliUtil,
 	)
 	return &App{}, nil
 }

Wire.go

@@ -0,0 +1,7 @@
+#!/bin/sh
+# This script is used as the command supplied to GIT_ASKPASS as a way to supply username/password
+# credentials to git, without having to use git credentials helpers, or having on-disk config.
+case "$1" in
+Username*) echo "${GIT_USERNAME}" ;;
+Password*) echo "${GIT_PASSWORD}" ;;
+esac

git-ask-pass.sh

@@ -46,6 +46,7 @@ type GitOpsConfig struct {
 	Token         string   `sql:"token"`
 	GitLabGroupId string   `sql:"gitlab_group_id"`
 	GitHubOrgId   string   `sql:"github_org_id"`
+	AzureProject  string   `sql:"azure_project"`
 	Host          string   `sql:"host"`
 	Active        bool     `sql:"active,notnull"`
 	models.AuditLog

internal/sql/repository/GitOpsConfigRepository.go

@@ -0,0 +1,105 @@
+package util
+
+import (
+	"fmt"
+	"go.uber.org/zap"
+	"gopkg.in/src-d/go-git.v4"
+	"gopkg.in/src-d/go-git.v4/config"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+type GitCliUtil struct {
+	logger *zap.SugaredLogger
+}
+
+func NewGitCliUtil(logger *zap.SugaredLogger) *GitCliUtil {
+	return &GitCliUtil{
+		logger: logger,
+	}
+}
+
+const GIT_ASK_PASS = "/git-ask-pass.sh"
+
+func (impl *GitCliUtil) Fetch(rootDir string, username string, password string) (response, errMsg string, err error) {
+	impl.logger.Debugw("git fetch ", "location", rootDir)
+	cmd := exec.Command("git", "-C", rootDir, "fetch", "origin", "--tags", "--force")
+	output, errMsg, err := impl.runCommandWithCred(cmd, username, password)
+	impl.logger.Debugw("fetch output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err)
+	return output, errMsg, err
+}
+
+func (impl *GitCliUtil) Pull(rootDir string, username string, password string, branch string) (response, errMsg string, err error) {
+	impl.logger.Debugw("git pull ", "location", rootDir)
+	cmd := exec.Command("git", "-C", rootDir, "pull", "origin", branch, "--force")
+	output, errMsg, err := impl.runCommandWithCred(cmd, username, password)
+	impl.logger.Debugw("pull output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err)
+	return output, errMsg, err
+}
+func (impl *GitCliUtil) Checkout(rootDir string, branch string) (response, errMsg string, err error) {
+	impl.logger.Debugw("git checkout ", "location", rootDir)
+	cmd := exec.Command("git", "-C", rootDir, "checkout", branch, "--force")
+	output, errMsg, err := impl.runCommand(cmd)
+	impl.logger.Debugw("checkout output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err)
+	return output, errMsg,  err
+}
+
+func (impl *GitCliUtil) runCommandWithCred(cmd *exec.Cmd, userName, password string) (response, errMsg string, err error) {
+	cmd.Env = append(os.Environ(),
+		fmt.Sprintf("GIT_ASKPASS=%s", GIT_ASK_PASS),
+		fmt.Sprintf("GIT_USERNAME=%s", userName),
+		fmt.Sprintf("GIT_PASSWORD=%s", password),
+	)
+	return impl.runCommand(cmd)
+}
+
+func (impl *GitCliUtil) runCommand(cmd *exec.Cmd) (response, errMsg string, err error) {
+	cmd.Env = append(cmd.Env, "HOME=/dev/null")
+	outBytes, err := cmd.CombinedOutput()
+	if err != nil {
+		exErr, ok := err.(*exec.ExitError)
+		if !ok {
+			return "", "", err
+		}
+		errOutput := string(exErr.Stderr)
+		return "", errOutput, err
+	}
+	output := string(outBytes)
+	output = strings.TrimSpace(output)
+	return output, "", nil
+}
+
+func (impl *GitCliUtil) Init(rootDir string, remoteUrl string, isBare bool) error {
+	//-----------------
+	err := os.RemoveAll(rootDir)
+	if err != nil {
+		impl.logger.Errorw("error in cleaning rootDir", "err", err)
+		return err
+	}
+	err = os.MkdirAll(rootDir, 0755)
+	if err != nil {
+		return err
+	}
+	repo, err := git.PlainInit(rootDir, isBare)
+	if err != nil {
+		return err
+	}
+	_, err = repo.CreateRemote(&config.RemoteConfig{
+		Name: git.DefaultRemoteName,
+		URLs: []string{remoteUrl},
+	})
+	return err
+}
+
+func (impl *GitCliUtil) Clone(rootDir string, remoteUrl string, username string, password string) (response, errMsg string, err error) {
+	err = impl.Init(rootDir, remoteUrl, false)
+	if err != nil {
+		return "", "", err
+	}
+	response, errMsg, err = impl.Fetch(rootDir, username, password)
+	if err == nil && errMsg == "" {
+		response, errMsg, err = impl.Pull(rootDir, username, password, "master")
+	}
+	return response, errMsg, err
+}

internal/util/GitCliUtil.go

@@ -28,7 +28,6 @@ import (
 	"golang.org/x/oauth2"
 	"gopkg.in/src-d/go-git.v4"
 	"gopkg.in/src-d/go-git.v4/plumbing/object"
-	"gopkg.in/src-d/go-git.v4/plumbing/transport"
 	"gopkg.in/src-d/go-git.v4/plumbing/transport/http"
 	"io/ioutil"
 	"net/url"
@@ -49,6 +48,7 @@ type GitFactory struct {
 	GitWorkingDir    string
 	logger           *zap.SugaredLogger
 	gitOpsRepository repository.GitOpsConfigRepository
+	gitCliUtil       *GitCliUtil
 }
 
 func (factory *GitFactory) Reload() error {
@@ -57,7 +57,7 @@ func (factory *GitFactory) Reload() error {
 	if err != nil {
 		return err
 	}
-	gitService := NewGitServiceImpl(cfg, logger)
+	gitService := NewGitServiceImpl(cfg, logger, factory.gitCliUtil)
 	factory.gitService = gitService
 	client, err := NewGitLabClient(cfg, logger, gitService)
 	if err != nil {
@@ -68,12 +68,12 @@ func (factory *GitFactory) Reload() error {
 	return nil
 }
 
-func NewGitFactory(logger *zap.SugaredLogger, gitOpsRepository repository.GitOpsConfigRepository) (*GitFactory, error) {
+func NewGitFactory(logger *zap.SugaredLogger, gitOpsRepository repository.GitOpsConfigRepository, gitCliUtil *GitCliUtil) (*GitFactory, error) {
 	cfg, err := GetGitConfig(gitOpsRepository)
 	if err != nil {
 		return nil, err
 	}
-	gitService := NewGitServiceImpl(cfg, logger)
+	gitService := NewGitServiceImpl(cfg, logger, gitCliUtil)
 	client, err := NewGitLabClient(cfg, logger, gitService)
 	if err != nil {
 		return nil, err
@@ -84,18 +84,21 @@ func NewGitFactory(logger *zap.SugaredLogger, gitOpsRepository repository.GitOps
 		gitService:       gitService,
 		gitOpsRepository: gitOpsRepository,
 		GitWorkingDir:    cfg.GitWorkingDir,
+		gitCliUtil:       gitCliUtil,
 	}, nil
 }
 
 type GitConfig struct {
-	GitlabGroupId      string                                                         //local
-	GitlabGroupPath    string                                                         //local
-	GitToken           string `env:"GIT_TOKEN" `                                      //not null  // public
-	GitUserName        string `env:"GIT_USERNAME" `                                   //not null  // public
-	GitWorkingDir      string `env:"GIT_WORKING_DIRECTORY" envDefault:"/tmp/gitops/"` //working directory for git. might use pvc
+	GitlabGroupId      string //local
+	GitlabGroupPath    string //local
+	GitToken           string //not null  // public
+	GitUserName        string //not null  // public
+	GitWorkingDir      string //working directory for git. might use pvc
 	GithubOrganization string
-	GitProvider        string `env:"GIT_PROVIDER" envDefault:"GITHUB"` // SUPPORTED VALUES  GITHUB, GITLAB
-	GitHost            string `env:"GIT_HOST" envDefault:""`
+	GitProvider        string // SUPPORTED VALUES  GITHUB, GITLAB
+	GitHost            string
+	AzureToken         string
+	AzureProject       string
 }
 
 func GetGitConfig(gitOpsRepository repository.GitOpsConfigRepository) (*GitConfig, error) {
@@ -121,6 +124,8 @@ func GetGitConfig(gitOpsRepository repository.GitOpsConfigRepository) (*GitConfi
 		GithubOrganization: gitOpsConfig.GitHubOrgId,
 		GitProvider:        gitOpsConfig.Provider,
 		GitHost:            gitOpsConfig.Host,
+		AzureToken:         gitOpsConfig.Token,
+		AzureProject:       gitOpsConfig.AzureProject,
 	}
 	return cfg, err
 }
@@ -190,9 +195,15 @@ func NewGitLabClient(config *GitConfig, logger *zap.SugaredLogger, gitService Gi
 			logger:     logger,
 			gitService: gitService,
 		}, nil
-	} else {
+	} else if config.GitProvider == "GITHUB" {
 		gitHubClient := NewGithubClient(config.GitToken, config.GithubOrganization, logger, gitService)
 		return gitHubClient, nil
+	} else if config.GitProvider == "AZURE_DEVOPS" {
+		gitAzureClient := NewGitAzureClient(config.AzureToken, config.GitHost, config.AzureProject, logger, gitService)
+		return gitAzureClient, nil
+	} else {
+		logger.Errorw("no gitops config provided, gitops will not work ")
+		return nil, nil
 	}
 }
 
@@ -373,17 +384,19 @@ type GitService interface {
 	Pull(repoRoot string) (err error)
 }
 type GitServiceImpl struct {
-	Auth   transport.AuthMethod
-	config *GitConfig
-	logger *zap.SugaredLogger
+	Auth       *http.BasicAuth
+	config     *GitConfig
+	logger     *zap.SugaredLogger
+	gitCliUtil *GitCliUtil
 }
 
-func NewGitServiceImpl(config *GitConfig, logger *zap.SugaredLogger) *GitServiceImpl {
+func NewGitServiceImpl(config *GitConfig, logger *zap.SugaredLogger, GitCliUtil *GitCliUtil) *GitServiceImpl {
 	auth := &http.BasicAuth{Password: config.GitToken, Username: config.GitUserName}
 	return &GitServiceImpl{
-		Auth:   auth,
-		logger: logger,
-		config: config,
+		Auth:       auth,
+		logger:     logger,
+		config:     config,
+		gitCliUtil: GitCliUtil,
 	}
 }
 
@@ -395,14 +408,14 @@ func (impl GitServiceImpl) GetCloneDirectory(targetDir string) (clonedDir string
 func (impl GitServiceImpl) Clone(url, targetDir string) (clonedDir string, err error) {
 	impl.logger.Debugw("git checkout ", "url", url, "dir", targetDir)
 	clonedDir = filepath.Join(impl.config.GitWorkingDir, targetDir)
-	_, err = git.PlainClone(clonedDir, false, &git.CloneOptions{
-		URL:  url,
-		Auth: impl.Auth,
-	})
+	_, errorMsg, err := impl.gitCliUtil.Clone(clonedDir, url, impl.Auth.Username, impl.Auth.Password)
 	if err != nil {
-		impl.logger.Errorw("error in git checkout ", "url", url, "targetDir", targetDir, "err", err)
+		impl.logger.Errorw("error in git checkout", "url", url, "targetDir", targetDir, "err", err)
 		return "", err
 	}
+	if errorMsg != "" {
+		return "", fmt.Errorf(errorMsg)
+	}
 	return clonedDir, nil
 }