Merge pull request #52 from fluxcd/git-improvements

Author: hiddeco

Dockerfile

@@ -20,7 +20,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o source-c
 
 FROM alpine:3.11
 
-RUN apk add --no-cache openssh-client ca-certificates tar tini 'git>=2.12.0' socat curl bash
+RUN apk add --no-cache ca-certificates tar tini 'git>=2.12.0' socat curl bash
 
 COPY --from=builder /workspace/source-controller /usr/local/bin/
 

controllers/gitrepository_controller.go

@@ -22,9 +22,7 @@ import (
 	"io/ioutil"
 	"os"
 
-	"github.com/blang/semver"
-	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/object"
 	"github.com/go-git/go-git/v5/plumbing/transport"
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
@@ -121,34 +119,18 @@ func (r *GitRepositoryReconciler) SetupWithManagerAndOptions(mgr ctrl.Manager, o
 }
 
 func (r *GitRepositoryReconciler) sync(ctx context.Context, repository sourcev1.GitRepository) (sourcev1.GitRepository, error) {
-	// set defaults: master branch, no tags fetching, max two commits
-	branch := "master"
-	revision := ""
-	tagMode := git.NoTags
-	depth := 2
-
-	// determine ref
-	refName := plumbing.NewBranchReferenceName(branch)
-	if repository.Spec.Reference != nil {
-		if repository.Spec.Reference.Branch != "" {
-			branch = repository.Spec.Reference.Branch
-			refName = plumbing.NewBranchReferenceName(branch)
-		}
-		if repository.Spec.Reference.Commit != "" {
-			depth = 0
-		} else {
-			if repository.Spec.Reference.Tag != "" {
-				refName = plumbing.NewTagReferenceName(repository.Spec.Reference.Tag)
-			}
-			if repository.Spec.Reference.SemVer != "" {
-				tagMode = git.AllTags
-			}
-		}
+	// create tmp dir for the Git clone
+	tmpGit, err := ioutil.TempDir("", repository.Name)
+	if err != nil {
+		err = fmt.Errorf("tmp dir error: %w", err)
+		return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
 	}
+	defer os.RemoveAll(tmpGit)
 
 	// determine auth method
 	var auth transport.AuthMethod
-	if repository.Spec.SecretRef != nil {
+	authStrategy := intgit.AuthSecretStrategyForURL(repository.Spec.URL)
+	if repository.Spec.SecretRef != nil && authStrategy != nil {
 		name := types.NamespacedName{
 			Namespace: repository.GetNamespace(),
 			Name:      repository.Spec.SecretRef.Name,
@@ -161,173 +143,32 @@ func (r *GitRepositoryReconciler) sync(ctx context.Context, repository sourcev1.
 			return sourcev1.GitRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err
 		}
 
-		method, cleanup, err := intgit.AuthMethodFromSecret(repository.Spec.URL, secret)
+		auth, err = authStrategy.Method(secret)
 		if err != nil {
 			err = fmt.Errorf("auth error: %w", err)
 			return sourcev1.GitRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err
 		}
-		if cleanup != nil {
-			defer cleanup()
-		}
-		auth = method
 	}
 
-	// create tmp dir for the Git clone
-	tmpGit, err := ioutil.TempDir("", repository.Name)
+	checkoutStrategy := intgit.CheckoutStrategyForRef(repository.Spec.Reference)
+	commit, revision, err := checkoutStrategy.Checkout(ctx, tmpGit, repository.Spec.URL, auth)
 	if err != nil {
-		err = fmt.Errorf("tmp dir error: %w", err)
-		return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
-	}
-	defer os.RemoveAll(tmpGit)
-
-	// clone to tmp
-	gitCtx, cancel := context.WithTimeout(ctx, repository.GetTimeout())
-	repo, err := git.PlainCloneContext(gitCtx, tmpGit, false, &git.CloneOptions{
-		URL:               repository.Spec.URL,
-		Auth:              auth,
-		RemoteName:        "origin",
-		ReferenceName:     refName,
-		SingleBranch:      true,
-		NoCheckout:        false,
-		Depth:             depth,
-		RecurseSubmodules: 0,
-		Progress:          nil,
-		Tags:              tagMode,
-	})
-	cancel()
-	if err != nil {
-		err = fmt.Errorf("git clone error: %w", err)
-		return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
-	}
-
-	// checkout commit or tag
-	if repository.Spec.Reference != nil {
-		if commit := repository.Spec.Reference.Commit; commit != "" {
-			w, err := repo.Worktree()
-			if err != nil {
-				err = fmt.Errorf("git worktree error: %w", err)
-				return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
-			}
-
-			err = w.Checkout(&git.CheckoutOptions{
-				Hash:  plumbing.NewHash(commit),
-				Force: true,
-			})
-			if err != nil {
-				err = fmt.Errorf("git checkout '%s' for '%s' error: %w", commit, branch, err)
-				return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
-			}
-		} else if exp := repository.Spec.Reference.SemVer; exp != "" {
-			rng, err := semver.ParseRange(exp)
-			if err != nil {
-				err = fmt.Errorf("semver parse range error: %w", err)
-				return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
-			}
-
-			repoTags, err := repo.Tags()
-			if err != nil {
-				err = fmt.Errorf("git list tags error: %w", err)
-				return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
-			}
-
-			tags := make(map[string]string)
-			_ = repoTags.ForEach(func(t *plumbing.Reference) error {
-				tags[t.Name().Short()] = t.Strings()[1]
-				return nil
-			})
-
-			svTags := make(map[string]string)
-			var svers []semver.Version
-			for tag, _ := range tags {
-				v, _ := semver.ParseTolerant(tag)
-				if rng(v) {
-					svers = append(svers, v)
-					svTags[v.String()] = tag
-				}
-			}
-
-			if len(svers) > 0 {
-				semver.Sort(svers)
-				v := svers[len(svers)-1]
-				t := svTags[v.String()]
-				commit := tags[t]
-				revision = fmt.Sprintf("%s/%s", t, commit)
-
-				w, err := repo.Worktree()
-				if err != nil {
-					err = fmt.Errorf("git worktree error: %w", err)
-					return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
-				}
-
-				err = w.Checkout(&git.CheckoutOptions{
-					Hash: plumbing.NewHash(commit),
-				})
-				if err != nil {
-					err = fmt.Errorf("git checkout error: %w", err)
-					return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
-				}
-			} else {
-				err = fmt.Errorf("no match found for semver: %s", repository.Spec.Reference.SemVer)
-				return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
-			}
-		}
-	}
-
-	// read commit hash
-	ref, err := repo.Head()
-	if err != nil {
-		err = fmt.Errorf("git resolve HEAD error: %w", err)
 		return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
 	}
 
 	// verify PGP signature
 	if repository.Spec.Verification != nil {
-		commit, err := repo.CommitObject(ref.Hash())
-		if err != nil {
-			err = fmt.Errorf("git resolve HEAD error: %w", err)
-			return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
-		}
-
-		if commit.PGPSignature == "" {
-			err = fmt.Errorf("PGP signature not found for commit '%s'", ref.Hash())
-			return sourcev1.GitRepositoryNotReady(repository, sourcev1.VerificationFailedReason, err.Error()), err
-		}
-
-		name := types.NamespacedName{
-			Namespace: repository.GetNamespace(),
+		err := r.verify(ctx, types.NamespacedName{
+			Namespace: repository.Namespace,
 			Name:      repository.Spec.Verification.SecretRef.Name,
-		}
-
-		var secret corev1.Secret
-		err = r.Client.Get(ctx, name, &secret)
+		}, commit)
 		if err != nil {
-			err = fmt.Errorf("PGP public keys secret error: %w", err)
-			return sourcev1.GitRepositoryNotReady(repository, sourcev1.VerificationFailedReason, err.Error()), err
-		}
-
-		var verified bool
-		for _, bytes := range secret.Data {
-			if _, err := commit.Verify(string(bytes)); err == nil {
-				verified = true
-				break
-			}
-		}
-
-		if !verified {
-			err = fmt.Errorf("PGP signature of '%s' can't be verified", commit.Author)
 			return sourcev1.GitRepositoryNotReady(repository, sourcev1.VerificationFailedReason, err.Error()), err
 		}
 	}
 
-	if revision == "" {
-		revision = fmt.Sprintf("%s/%s", branch, ref.Hash().String())
-		if repository.Spec.Reference != nil && repository.Spec.Reference.Tag != "" {
-			revision = fmt.Sprintf("%s/%s", repository.Spec.Reference.Tag, ref.Hash().String())
-		}
-	}
-
 	artifact := r.Storage.ArtifactFor(repository.Kind, repository.ObjectMeta.GetObjectMeta(),
-		fmt.Sprintf("%s.tar.gz", ref.Hash().String()), revision)
+		fmt.Sprintf("%s.tar.gz", commit.Hash.String()), revision)
 
 	// create artifact dir
 	err = r.Storage.MkdirAll(artifact)
@@ -388,6 +229,29 @@ func (r *GitRepositoryReconciler) shouldResetStatus(repository sourcev1.GitRepos
 	}
 }
 
+func (r *GitRepositoryReconciler) verify(ctx context.Context, publicKeySecret types.NamespacedName, commit *object.Commit) error {
+	if commit.PGPSignature == "" {
+		return fmt.Errorf("no PGP signature found for commit: %s", commit.Hash)
+	}
+
+	var secret corev1.Secret
+	if err := r.Client.Get(ctx, publicKeySecret, &secret); err != nil {
+		return fmt.Errorf("PGP public keys secret error: %w", err)
+	}
+
+	var verified bool
+	for _, bytes := range secret.Data {
+		if _, err := commit.Verify(string(bytes)); err == nil {
+			verified = true
+			break
+		}
+	}
+	if !verified {
+		return fmt.Errorf("PGP signature '%s' of '%s' can't be verified", commit.PGPSignature, commit.Author)
+	}
+	return nil
+}
+
 // gc performs a garbage collection on all but current artifacts of
 // the given repository.
 func (r *GitRepositoryReconciler) gc(repository sourcev1.GitRepository) error {

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/go-logr/logr v0.1.0
 	github.com/onsi/ginkgo v1.11.0
 	github.com/onsi/gomega v1.8.1
-	github.com/pkg/errors v0.9.1
+	golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073
 	helm.sh/helm/v3 v3.1.2
 	k8s.io/api v0.17.2
 	k8s.io/apimachinery v0.17.2