Merge pull request #213 from phillebaba/feature/git2go

Add git2go option to enable Azure DevOps

Author: Philip Laine

.github/actions/run-tests/Dockerfile

@@ -1,7 +1,8 @@
 FROM golang:1.15-alpine
 
 # Add any build or testing essential system packages
-RUN apk add --no-cache build-base git
+RUN apk add --no-cache build-base git pkgconf
+RUN apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/community libgit2-dev=1.1.0-r1
 
 # Use the GitHub Actions uid:gid combination for proper fs permissions
 RUN addgroup -g 116 -S test && adduser -u 1001 -S -g test test

CONTRIBUTING.md

@@ -28,6 +28,21 @@ to join the conversation (this will also add an invitation to your
 Google calendar for our [Flux
 meeting](https://docs.google.com/document/d/1l_M0om0qUEN_NNiGgpqJ2tvsF2iioHkaARDeh6b70B0/edit#)).
 
+### Installing required dependencies
+
+The dependency [libgit2](https://libgit2.org/) needs to be installed to be able to run
+Source Controller or its test-suite locally (not in a container).
+
+**macOS**
+```
+brew install libgit2
+```
+
+**Arch Linux**
+```
+pacman -S libgit2
+```
+
 ### How to run the test suite
 
 You can run the unit tests by simply doing

Dockerfile

@@ -1,6 +1,9 @@
 # Docker buildkit multi-arch build requires golang alpine
 FROM golang:1.15-alpine as builder
 
+RUN apk add gcc pkgconfig libc-dev
+RUN apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/community libgit2-dev=1.1.0-r1
+
 WORKDIR /workspace
 
 # copy api submodule
@@ -20,14 +23,15 @@ COPY pkg/ pkg/
 COPY internal/ internal/
 
 # build without specifing the arch
-RUN CGO_ENABLED=0 go build -a -o source-controller main.go
+RUN CGO_ENABLED=1 go build -o source-controller main.go
 
 FROM alpine:3.12
 
 # link repo to the GitHub Container Registry image
 LABEL org.opencontainers.image.source="https://github.com/fluxcd/source-controller"
 
 RUN apk add --no-cache ca-certificates tini
+RUN apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/community libgit2=1.1.0-r1
 
 COPY --from=builder /workspace/source-controller /usr/local/bin/
 

Makefile

@@ -39,7 +39,7 @@ deploy: manifests
 	kustomize build config/default | kubectl apply -f -
 
 # Deploy controller dev image in the configured Kubernetes cluster in ~/.kube/config
-dev-deploy: manifests
+dev-deploy:
 	mkdir -p config/dev && cp config/default/* config/dev
 	cd config/dev && kustomize edit set image fluxcd/source-controller=${IMG}
 	kustomize build config/dev | kubectl apply -f -

api/v1beta1/gitrepository_types.go

@@ -26,6 +26,10 @@ import (
 const (
 	// GitRepositoryKind is the string representation of a GitRepository.
 	GitRepositoryKind = "GitRepository"
+	// Represents the go-git git implementation kind
+	GoGitImplementation = "go-git"
+	// Represents the gi2go git implementation kind
+	LibGit2Implementation = "libgit2"
 )
 
 // GitRepositorySpec defines the desired state of a Git repository.
@@ -70,6 +74,13 @@ type GitRepositorySpec struct {
 	// This flag tells the controller to suspend the reconciliation of this source.
 	// +optional
 	Suspend bool `json:"suspend,omitempty"`
+
+	// Determines which git client library to use.
+	// Defaults to go-git, valid values are ('go-git', 'libgit2').
+	// +kubebuilder:validation:Enum=go-git;libgit2
+	// +kubebuilder:default:=go-git
+	// +optional
+	GitImplementation string `json:"gitImplementation,omitempty"`
 }
 
 // GitRepositoryRef defines the Git ref used for pull and checkout operations.

config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml

@@ -49,6 +49,14 @@ spec:
           spec:
             description: GitRepositorySpec defines the desired state of a Git repository.
             properties:
+              gitImplementation:
+                default: go-git
+                description: Determines which git client library to use. Defaults
+                  to go-git, valid values are ('go-git', 'libgit2').
+                enum:
+                - go-git
+                - libgit2
+                type: string
               ignore:
                 description: Ignore overrides the set of excluded patterns in the
                   .sourceignore format (which is the same as .gitignore). If not provided,

controllers/gitrepository_controller.go

@@ -25,8 +25,6 @@ import (
 	"time"
 
 	"github.com/fluxcd/pkg/apis/meta"
-	"github.com/go-git/go-git/v5/plumbing/object"
-	"github.com/go-git/go-git/v5/plumbing/transport"
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
@@ -46,6 +44,7 @@ import (
 
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
 	"github.com/fluxcd/source-controller/pkg/git"
+	"github.com/fluxcd/source-controller/pkg/git/common"
 )
 
 // GitRepositoryReconciler reconciles a GitRepository object
@@ -154,7 +153,6 @@ func (r *GitRepositoryReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
 	))
 
 	return ctrl.Result{RequeueAfter: repository.GetInterval().Duration}, nil
-
 }
 
 type GitRepositoryReconcilerOptions struct {
@@ -183,9 +181,9 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, repository sour
 	defer os.RemoveAll(tmpGit)
 
 	// determine auth method
-	var auth transport.AuthMethod
+	auth := &common.Auth{}
 	if repository.Spec.SecretRef != nil {
-		authStrategy, err := git.AuthSecretStrategyForURL(repository.Spec.URL)
+		authStrategy, err := git.AuthSecretStrategyForURL(repository.Spec.URL, repository.Spec.GitImplementation)
 		if err != nil {
 			return sourcev1.GitRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err
 		}
@@ -209,14 +207,17 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, repository sour
 		}
 	}
 
-	checkoutStrategy := git.CheckoutStrategyForRef(repository.Spec.Reference)
+	checkoutStrategy, err := git.CheckoutStrategyForRef(repository.Spec.Reference, repository.Spec.GitImplementation)
+	if err != nil {
+		return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
+	}
 	commit, revision, err := checkoutStrategy.Checkout(ctx, tmpGit, repository.Spec.URL, auth)
 	if err != nil {
 		return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
 	}
 
 	// return early on unchanged revision
-	artifact := r.Storage.NewArtifactFor(repository.Kind, repository.GetObjectMeta(), revision, fmt.Sprintf("%s.tar.gz", commit.Hash.String()))
+	artifact := r.Storage.NewArtifactFor(repository.Kind, repository.GetObjectMeta(), revision, fmt.Sprintf("%s.tar.gz", commit.Hash()))
 	if apimeta.IsStatusConditionTrue(repository.Status.Conditions, meta.ReadyCondition) && repository.GetArtifact().HasRevision(artifact.Revision) {
 		if artifact.URL != repository.GetArtifact().URL {
 			r.Storage.SetArtifactURL(repository.GetArtifact())
@@ -227,10 +228,17 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, repository sour
 
 	// verify PGP signature
 	if repository.Spec.Verification != nil {
-		err := r.verify(ctx, types.NamespacedName{
+		publicKeySecret := types.NamespacedName{
 			Namespace: repository.Namespace,
 			Name:      repository.Spec.Verification.SecretRef.Name,
-		}, commit)
+		}
+		var secret corev1.Secret
+		if err := r.Client.Get(ctx, publicKeySecret, &secret); err != nil {
+			err = fmt.Errorf("PGP public keys secret error: %w", err)
+			return sourcev1.GitRepositoryNotReady(repository, sourcev1.VerificationFailedReason, err.Error()), err
+		}
+
+		err := commit.Verify(secret)
 		if err != nil {
 			return sourcev1.GitRepositoryNotReady(repository, sourcev1.VerificationFailedReason, err.Error()), err
 		}
@@ -288,30 +296,6 @@ func (r *GitRepositoryReconciler) reconcileDelete(ctx context.Context, repositor
 	return ctrl.Result{}, nil
 }
 
-// verify returns an error if the PGP signature can't be verified
-func (r *GitRepositoryReconciler) verify(ctx context.Context, publicKeySecret types.NamespacedName, commit *object.Commit) error {
-	if commit.PGPSignature == "" {
-		return fmt.Errorf("no PGP signature found for commit: %s", commit.Hash)
-	}
-
-	var secret corev1.Secret
-	if err := r.Client.Get(ctx, publicKeySecret, &secret); err != nil {
-		return fmt.Errorf("PGP public keys secret error: %w", err)
-	}
-
-	var verified bool
-	for _, bytes := range secret.Data {
-		if _, err := commit.Verify(string(bytes)); err == nil {
-			verified = true
-			break
-		}
-	}
-	if !verified {
-		return fmt.Errorf("PGP signature '%s' of '%s' can't be verified", commit.PGPSignature, commit.Author)
-	}
-	return nil
-}
-
 // resetStatus returns a modified v1beta1.GitRepository and a boolean indicating
 // if the status field has been reset.
 func (r *GitRepositoryReconciler) resetStatus(repository sourcev1.GitRepository) (sourcev1.GitRepository, bool) {

controllers/gitrepository_controller_test.go

@@ -86,6 +86,8 @@ var _ = Describe("GitRepositoryReconciler", func() {
 			expectStatus   metav1.ConditionStatus
 			expectMessage  string
 			expectRevision string
+
+			gitImplementation string
 		}
 
 		DescribeTable("Git references tests", func(t refTestCase) {
@@ -262,5 +264,65 @@ var _ = Describe("GitRepositoryReconciler", func() {
 				expectMessage: "git commit 'invalid' not found: object not found",
 			}),
 		)
+
+		DescribeTable("Git self signed cert tests", func(t refTestCase) {
+			err = gitServer.StartHTTPS(examplePublicKey, examplePrivateKey, exampleCA, "example.com")
+			defer gitServer.StopHTTP()
+			Expect(err).NotTo(HaveOccurred())
+
+			u, err := url.Parse(gitServer.HTTPAddress())
+			Expect(err).NotTo(HaveOccurred())
+			u.Path = path.Join(u.Path, fmt.Sprintf("repository-%s.git", randStringRunes(5)))
+
+			key := types.NamespacedName{
+				Name:      fmt.Sprintf("git-ref-test-%s", randStringRunes(5)),
+				Namespace: namespace.Name,
+			}
+			created := &sourcev1.GitRepository{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      key.Name,
+					Namespace: key.Namespace,
+				},
+				Spec: sourcev1.GitRepositorySpec{
+					URL:               u.String(),
+					Interval:          metav1.Duration{Duration: indexInterval},
+					Reference:         t.reference,
+					GitImplementation: t.gitImplementation,
+				},
+			}
+			Expect(k8sClient.Create(context.Background(), created)).Should(Succeed())
+			defer k8sClient.Delete(context.Background(), created)
+
+			got := &sourcev1.GitRepository{}
+			var cond metav1.Condition
+			Eventually(func() bool {
+				_ = k8sClient.Get(context.Background(), key, got)
+				for _, c := range got.Status.Conditions {
+					if c.Reason == t.waitForReason {
+						cond = c
+						return true
+					}
+				}
+				return false
+			}, timeout, interval).Should(BeTrue())
+
+			Expect(cond.Status).To(Equal(t.expectStatus))
+			Expect(cond.Message).To(ContainSubstring(t.expectMessage))
+			Expect(got.Status.Artifact == nil).To(Equal(t.expectRevision == ""))
+		},
+			Entry("self signed v1", refTestCase{
+				reference:     &sourcev1.GitRepositoryRef{Branch: "main"},
+				waitForReason: sourcev1.GitOperationFailedReason,
+				expectStatus:  metav1.ConditionFalse,
+				expectMessage: "x509: certificate signed by unknown authority",
+			}),
+			Entry("self signed v2", refTestCase{
+				reference:         &sourcev1.GitRepositoryRef{Branch: "main"},
+				waitForReason:     sourcev1.GitOperationFailedReason,
+				expectStatus:      metav1.ConditionFalse,
+				expectMessage:     "error: user rejected certificate",
+				gitImplementation: sourcev1.LibGit2Implementation,
+			}),
+		)
 	})
 })

docs/api/source.md

@@ -387,6 +387,19 @@ bool
 <p>This flag tells the controller to suspend the reconciliation of this source.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>gitImplementation</code><br>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Determines which git client library to use.
+Defaults to go-git, valid values are (&lsquo;go-git&rsquo;, &lsquo;libgit2&rsquo;).</p>
+</td>
+</tr>
 </table>
 </td>
 </tr>
@@ -1220,6 +1233,19 @@ bool
 <p>This flag tells the controller to suspend the reconciliation of this source.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>gitImplementation</code><br>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Determines which git client library to use.
+Defaults to go-git, valid values are (&lsquo;go-git&rsquo;, &lsquo;libgit2&rsquo;).</p>
+</td>
+</tr>
 </tbody>
 </table>
 </div>