Merge pull request #468 from fluxcd/remove-libgit2

Remove libgit2 and git2go from codebase

Author: Paulo Gomes

.dockerignore

@@ -1 +1 @@
-build/libgit2/
+build/

.gitignore

@@ -29,5 +29,5 @@ testbin
 *.swo
 *~
 
-# Exclude all libgit2 related files
+# Exclude all build related files
 build/

ATTRIBUTIONS.md

@@ -13,80 +13,7 @@ There are a number of dependencies required to be able to run the controller and
 - [Install Docker](https://docs.docker.com/engine/install/)
 - (Optional) [Install Kubebuilder](https://book.kubebuilder.io/quick-start.html#installation)
 
-The dependency [libgit2](https://libgit2.org/) also needs to be installed to be able
-to run `image-automation-controller` or its test-suite locally (not in a container).
-
-In case this dependency is not present on your system (at the expected
-version), the first invocation of a `make` target that requires the
-dependency will attempt to compile it locally to `hack/libgit2`. For this build
-to succeed ensure the following dependencies are present on your system:
-- [CMake](https://cmake.org/download/)
-- [OpenSSL 1.1](https://www.openssl.org/source/)
-- [LibSSH2](https://www.libssh2.org/)
-- [pkg-config](https://freedesktop.org/wiki/Software/pkg-config/)
-
-
-Triggering a manual build of the dependency is possible as well by running
-`make libgit2`. To enforce the build, for example if your system dependencies
-match but are not linked in a compatible way, append `LIBGIT2_FORCE=1` to the
-`make` command.
-
-Follow the instructions below to install these dependencies to your system.
-
-### macOS
-
-```console
-$ # Ensure libgit2 dependencies are available
-$ brew install cmake [email protected] libssh2 pkg-config
-$ LIBGIT2_FORCE=1 make libgit2
-```
-
-#### openssl and pkg-config
-
-You may see this message when trying to run a build:
-
-```
-# pkg-config --cflags libgit2
-Package openssl was not found in the pkg-config search path.
-Perhaps you should add the directory containing `openssl.pc'
-to the PKG_CONFIG_PATH environment variable
-Package 'openssl', required by 'libgit2', not found
-pkg-config: exit status 1
-```
-
-On some macOS systems `brew` will not link the pkg-config file for
-openssl into expected directory, because it clashes with a
-system-provided openssl. When installing it, or if you do
-
-```console
-brew link [email protected]
-Warning: Refusing to link macOS provided/shadowed software: [email protected]
-If you need to have [email protected] first in your PATH, run:
-  echo 'export PATH="/usr/local/opt/[email protected]/bin:$PATH"' >> ~/.profile
-
-For compilers to find [email protected] you may need to set:
-  export LDFLAGS="-L/usr/local/opt/[email protected]/lib"
-  export CPPFLAGS="-I/usr/local/opt/[email protected]/include"
-
-For pkg-config to find [email protected] you may need to set:
-  export PKG_CONFIG_PATH="/usr/local/opt/[email protected]/lib/pkgconfig"
-```
-
-.. it tells you (in the last part of the message) how to add `openssl`
-to your `PKG_CONFIG_PATH` so it can be found.
-
-### Linux
-
-```console
-$ # Ensure libgit2 dependencies are available
-$ pacman -S cmake openssl libssh2
-$ LIBGIT2_FORCE=1 make libgit2
-```
-
-**Note:** Example shown is for Arch Linux, but likewise procedure can be
-followed using any other package manager, e.g. `apt`.
-
-In addition to the above, the following dependencies are also used by some of the `make` targets:
+The following dependencies are also used by some of the `make` targets:
 
 - `controller-gen` (v0.7.0)
 - `gen-crd-api-reference-docs` (v0.3.0)
@@ -169,18 +96,11 @@ Create a `.vscode/launch.json` file:
             "type": "go",
             "request": "launch",
             "mode": "auto",
-            "envFile": "${workspaceFolder}/build/.env",
             "program": "${workspaceFolder}/main.go"
         }
     ]
 }
 ```
 
-Create the environment file containing details on how to load 
-`libgit2` dependencies:
-```bash
-make env
-```
-
 Start debugging by either clicking `Run` > `Start Debugging` or using
 the relevant shortcut.

DEVELOPMENT.md

@@ -2,11 +2,6 @@ ARG BASE_VARIANT=alpine
 ARG GO_VERSION=1.19
 ARG XX_VERSION=1.1.2
 
-ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2-only
-ARG LIBGIT2_TAG=v0.4.0
-
-FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} AS libgit2-libs
-
 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
 
 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as gostable
@@ -17,12 +12,18 @@ FROM gostable AS go-linux
 # These will be used at current arch to yield execute the cross compilations.
 FROM go-${TARGETOS} AS build-base
 
-RUN apk add clang lld pkgconfig
+RUN apk add clang lld
 
 COPY --from=xx / /
 
-# build-go-mod can still be cached at build platform architecture.
-FROM build-base as build-go-mod
+# build can still be cached at build platform architecture.
+FROM build-base as build
+
+ARG TARGETPLATFORM
+
+# Some dependencies have to installed for the target platform:
+# https://github.com/tonistiigi/xx#go--cgo
+RUN xx-apk add musl-dev gcc
 
 # Configure workspace
 WORKDIR /workspace
@@ -37,20 +38,6 @@ COPY go.sum go.sum
 # Cache modules
 RUN go mod download
 
-# Build stage install per target platform
-# dependency and effectively cross compile the application.
-FROM build-go-mod as build
-
-ARG TARGETPLATFORM
-
-COPY --from=libgit2-libs /usr/local/ /usr/local/
-
-# Some dependencies have to installed
-# for the target platform: https://github.com/tonistiigi/xx#go--cgo
-RUN xx-apk add musl-dev gcc clang lld
-
-WORKDIR /workspace
-
 # Copy source code
 COPY main.go main.go
 COPY controllers/ controllers/
@@ -59,11 +46,13 @@ COPY internal/ internal/
 
 ARG TARGETPLATFORM
 ARG TARGETARCH
+
+# Reasons why CGO is in use:
+# - The SHA1 implementation (sha1cd) used by go-git depends on CGO for
+#   performance reasons. See: https://github.com/pjbgf/sha1cd/issues/15
 ENV CGO_ENABLED=1
 
-RUN export LIBRARY_PATH="/usr/local/$(xx-info triple)" && \
-  export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig" && \
-  export CGO_LDFLAGS="$(pkg-config --static --libs --cflags libgit2) -static -fuse-ld=lld" && \
+RUN export CGO_LDFLAGS="-static -fuse-ld=lld" && \
   xx-go build  \
   -ldflags "-s -w" \
   -tags 'netgo,osusergo,static_build' \
@@ -80,7 +69,6 @@ RUN apk --no-cache add ca-certificates \
 
 # Copy over binary from build
 COPY --from=build /image-automation-controller /usr/local/bin/
-COPY ATTRIBUTIONS.md /
 
 USER 65534:65534
 ENTRYPOINT [ "image-automation-controller" ]

Dockerfile

@@ -6,10 +6,6 @@ TAG ?= latest
 # Produce CRDs that work back to Kubernetes 1.16
 CRD_OPTIONS ?= crd:crdVersions=v1
 
-# Base image used to build the Go binary
-LIBGIT2_IMG ?= ghcr.io/fluxcd/golang-with-libgit2-only
-LIBGIT2_TAG ?= v0.4.0
-
 # Allows for defining additional Docker buildx arguments,
 # e.g. '--push'.
 BUILD_ARGS ?=
@@ -44,19 +40,6 @@ ENVTEST_BIN_VERSION ?= 1.19.2
 # each fuzzer should run for.
 FUZZ_TIME ?= 1m
 
-# Caches libgit2 versions per tag, "forcing" rebuild only when needed.
-LIBGIT2_PATH := $(BUILD_DIR)/libgit2/$(LIBGIT2_TAG)
-LIBGIT2_LIB_PATH := $(LIBGIT2_PATH)/lib
-LIBGIT2_LIB64_PATH := $(LIBGIT2_PATH)/lib64
-LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.a
-
-export CGO_ENABLED=1
-export PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig
-export LIBRARY_PATH=$(LIBGIT2_LIB_PATH)
-export CGO_CFLAGS=-I$(LIBGIT2_PATH)/include -I$(LIBGIT2_PATH)/include/openssl
-export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libgit2 2>/dev/null)
-
-# The pkg-config command will yield warning messages until libgit2 is downloaded.
 ifeq ($(shell uname -s),Darwin)
 GO_STATIC_FLAGS=-ldflags "-s -w" -tags 'netgo,osusergo,static_build'
 endif
@@ -129,17 +112,17 @@ ifeq ($(shell uname -s),Darwin)
 endif
 
 KUBEBUILDER_ASSETS?="$(shell $(ENVTEST) --arch=$(ENVTEST_ARCH) use -i $(ENVTEST_KUBERNETES_VERSION) --bin-dir=$(ENVTEST_ASSETS_DIR) -p path)"
-test: $(LIBGIT2) tidy test-api test_deps generate fmt vet manifests api-docs install-envtest ## Run tests
+test: tidy test-api test_deps generate fmt vet manifests api-docs install-envtest ## Run tests
 	KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) \
 	go test $(GO_STATIC_FLAGS) $(GO_TEST_ARGS) ./... -coverprofile cover.out
 
 test-api:	## Run api tests
 	cd api; go test $(GO_TEST_ARGS) ./... -coverprofile cover.out
 
-manager: $(LIBGIT2) generate fmt vet	## Build manager binary
+manager: generate fmt vet	## Build manager binary
 	go build -o $(BUILD_DIR)/bin/manager ./main.go
 
-run: $(LIBGIT2) generate fmt vet manifests	# Run against the configured Kubernetes cluster in ~/.kube/config
+run: generate fmt vet manifests	# Run against the configured Kubernetes cluster in ~/.kube/config
 	go run $(GO_STATIC_FLAGS) ./main.go --log-level=${LOG_LEVEL} --log-encoding=console
 
 install: manifests	## Install CRDs into a cluster
@@ -172,7 +155,7 @@ fmt:	## Run go fmt against code
 	go fmt ./...
 	cd api; go fmt ./...
 
-vet: $(LIBGIT2)	## Run go vet against code
+vet: ## Run go vet against code
 	go vet ./...
 	cd api; go vet ./...
 
@@ -182,8 +165,6 @@ generate: controller-gen	## Generate code
 
 docker-build:  ## Build the Docker image
 	docker buildx build \
-		--build-arg LIBGIT2_IMG=$(LIBGIT2_IMG) \
-		--build-arg LIBGIT2_TAG=$(LIBGIT2_TAG) \
 		--platform=$(BUILD_PLATFORMS) \
 		-t $(IMG):$(TAG) \
 		$(BUILD_ARGS) .
@@ -200,14 +181,6 @@ CONTROLLER_GEN = $(GOBIN)/controller-gen
 controller-gen: ## Download controller-gen locally if necessary.
 	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/[email protected])
 
-libgit2: $(LIBGIT2)  ## Detect or download libgit2 library
-
-COSIGN = $(GOBIN)/cosign
-$(LIBGIT2):
-	$(call go-install-tool,$(COSIGN),github.com/sigstore/cosign/cmd/cosign@latest)
-
-	IMG=$(LIBGIT2_IMG) TAG=$(LIBGIT2_TAG) PATH=$(PATH):$(GOBIN) ./hack/install-libraries.sh
-
 # Find or download gen-crd-api-reference-docs
 GEN_CRD_API_REFERENCE_DOCS = $(GOBIN)/gen-crd-api-reference-docs
 .PHONY: gen-crd-api-reference-docs
@@ -227,7 +200,7 @@ setup-envtest: ## Download envtest-setup locally if necessary.
 	$(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest)
 
 # Build fuzzers used by oss-fuzz.
-fuzz-build: $(LIBGIT2)
+fuzz-build:
 	rm -rf $(shell pwd)/build/fuzz/
 	mkdir -p $(shell pwd)/build/fuzz/out/
 
@@ -268,20 +241,7 @@ endef
 update-attributions:
 	./hack/update-attributions.sh
 
-verify: update-attributions fmt
-ifneq ($(shell grep -o 'LIBGIT2_IMG ?= \w.*' Makefile | cut -d ' ' -f 3):$(shell grep -o 'LIBGIT2_TAG ?= \w.*' Makefile | cut -d ' ' -f 3), \
-		$(shell grep -o "LIBGIT2_IMG=\w.*" Dockerfile | cut -d'=' -f2):$(shell grep -o "LIBGIT2_TAG=\w.*" Dockerfile | cut -d'=' -f2))
-	@{ \
-	echo "LIBGIT2_IMG and LIBGIT2_TAG must match in both Makefile and Dockerfile"; \
-	exit 1; \
-	}
-endif
-ifneq ($(shell grep -o 'LIBGIT2_TAG ?= \w.*' Makefile | cut -d ' ' -f 3), $(shell grep -o "LIBGIT2_TAG=.*" tests/fuzz/oss_fuzz_prebuild.sh | sed 's;LIBGIT2_TAG="$${LIBGIT2_TAG:-;;g' | sed 's;}";;g'))
-	@{ \
-	echo "LIBGIT2_TAG must match in both Makefile and tests/fuzz/oss_fuzz_prebuild.sh"; \
-	exit 1; \
-	}
-endif
+verify:
 ifneq (, $(shell git status --porcelain --untracked-files=no))
 	@{ \
 	echo "working directory is dirty:"; \
@@ -293,15 +253,3 @@ endif
 .PHONY: help
 help:  ## Display this help menu
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
-
-# Creates an env file that can be used to load all image-automation-controller's
-# dependencies this is handy when you want to run adhoc debug sessions on tests or
-# start the controller in a new debug session.
-env: $(LIBGIT2)
-	echo 'GO_ENABLED="1"' > $(BUILD_DIR)/.env
-	echo 'PKG_CONFIG_PATH="$(PKG_CONFIG_PATH)"' >> $(BUILD_DIR)/.env
-	echo 'LIBRARY_PATH="$(LIBRARY_PATH)"' >> $(BUILD_DIR)/.env
-	echo 'CGO_CFLAGS="$(CGO_CFLAGS)"' >> $(BUILD_DIR)/.env
-	echo 'CGO_LDFLAGS="$(CGO_LDFLAGS)"' >> $(BUILD_DIR)/.env
-	echo 'KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS)' >> $(BUILD_DIR)/.env
-	echo 'GIT_CONFIG_GLOBAL=/dev/null' >> $(BUILD_DIR)/.env

Makefile

@@ -21,11 +21,5 @@ guide](https://fluxcd.io/flux/guides/image-update/).
 
 ## How to work on it
 
-The shared library `libgit2` needs to be installed to test or build
-locally. The version required corresponds to the version of git2go
-(which are Go bindings for libgit2), according to [this
-table](https://github.com/libgit2/git2go#which-go-version-to-use).
-
-See
-https://github.com/fluxcd/source-controller/blob/main/DEVELOPMENT.md#installing-required-dependencies
-for instructions on how to install `libgit2`.
+For additional information on dependecies and how to contribute
+please refer to [DEVELOPMENT.md](DEVELOPMENT.md).

README.md

@@ -52,7 +52,6 @@ import (
 	"github.com/fluxcd/pkg/apis/meta"
 	"github.com/fluxcd/pkg/git"
 	"github.com/fluxcd/pkg/git/gogit"
-	"github.com/fluxcd/pkg/git/libgit2"
 	"github.com/fluxcd/pkg/git/repository"
 	"github.com/fluxcd/pkg/runtime/acl"
 	helper "github.com/fluxcd/pkg/runtime/controller"
@@ -258,33 +257,16 @@ func (r *ImageUpdateAutomationReconciler) Reconcile(ctx context.Context, req ctr
 		return failWithError(err)
 	}
 
-	gitImplementation := origin.Spec.GitImplementation
-	if goGitOnly, _ := r.features[features.ForceGoGitImplementation]; goGitOnly {
-		gitImplementation = sourcev1.GoGitImplementation
+	clientOpts := []gogit.ClientOption{gogit.WithDiskStorage()}
+	forcePush, _ := features.Enabled(features.GitForcePushBranch)
+	if forcePush && pushBranch != ref.Branch {
+		clientOpts = append(clientOpts, gogit.WithForcePush())
 	}
-
-	var gitClient repository.Client
-	switch gitImplementation {
-	case sourcev1.LibGit2Implementation:
-		clientOpts := []libgit2.ClientOption{libgit2.WithDiskStorage()}
-		if authOpts.Transport == git.HTTP {
-			clientOpts = append(clientOpts, libgit2.WithInsecureCredentialsOverHTTP())
-		}
-		gitClient, err = libgit2.NewClient(tmp, authOpts, clientOpts...)
-	case sourcev1.GoGitImplementation, "":
-		clientOpts := []gogit.ClientOption{gogit.WithDiskStorage()}
-		forcePush, _ := features.Enabled(features.GitForcePushBranch)
-		if forcePush && pushBranch != ref.Branch {
-			clientOpts = append(clientOpts, gogit.WithForcePush())
-		}
-		if authOpts.Transport == git.HTTP {
-			clientOpts = append(clientOpts, gogit.WithInsecureCredentialsOverHTTP())
-		}
-
-		gitClient, err = gogit.NewClient(tmp, authOpts, clientOpts...)
-	default:
-		err = fmt.Errorf("failed to create git client; referred GitRepository has invalid implementation: %s", gitImplementation)
+	if authOpts.Transport == git.HTTP {
+		clientOpts = append(clientOpts, gogit.WithInsecureCredentialsOverHTTP())
 	}
+
+	gitClient, err := gogit.NewClient(tmp, authOpts, clientOpts...)
 	if err != nil {
 		return failWithError(err)
 	}