Merge pull request #977 from fluxcd/remove-libgit2

Remove libgit2 and git2go from codebase

Author: Paulo Gomes

.dockerignore

@@ -1 +1 @@
-build/libgit2/
+build/

ATTRIBUTIONS.md

@@ -13,27 +13,6 @@ There are a number of dependencies required to be able to run the controller and
 - [Install Docker](https://docs.docker.com/engine/install/)
 - (Optional) [Install Kubebuilder](https://book.kubebuilder.io/quick-start.html#installation)
 
-The [libgit2](https://libgit2.org/) dependency is now automatically managed by the Makefile logic.
-However, it depends on [pkg-config](https://freedesktop.org/wiki/Software/pkg-config/) being installed:
-
-### macOS
-
-```console
-$ # Ensure pkg-config is installed
-$ brew install pkg-config
-```
-
-### Linux
-
-```console
-$ # Ensure pkg-config is installed
-$ pacman -S pkgconf
-```
-
-**Note:** Example shown is for Arch Linux, but likewise procedure can be
-followed using any other package manager. Some distributions may have slight 
-variation of package names (e.g. `apt install -y pkg-config`).
-
 In addition to the above, the following dependencies are also used by some of the `make` targets:
 
 - `controller-gen` (v0.7.0)
@@ -149,18 +128,11 @@ Create a `.vscode/launch.json` file:
             "type": "go",
             "request": "launch",
             "mode": "auto",
-            "envFile": "${workspaceFolder}/build/.env",
             "program": "${workspaceFolder}/main.go"
         }
     ]
 }
 ```
 
-Create the environment file containing details on how to load 
-`libgit2` dependencies:
-```bash
-make env
-```
-
 Start debugging by either clicking `Run` > `Start Debugging` or using
 the relevant shortcut.

DEVELOPMENT.md

@@ -2,11 +2,6 @@ ARG BASE_VARIANT=alpine
 ARG GO_VERSION=1.19
 ARG XX_VERSION=1.1.2
 
-ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2-only
-ARG LIBGIT2_TAG=v0.4.0
-
-FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} AS libgit2-libs
-
 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
 
 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as gostable
@@ -17,12 +12,18 @@ FROM gostable AS go-linux
 # These will be used at current arch to yield execute the cross compilations.
 FROM go-${TARGETOS} AS build-base
 
-RUN apk add --no-cache clang lld pkgconfig
+RUN apk add --no-cache clang lld
 
 COPY --from=xx / /
 
 # build-go-mod can still be cached at build platform architecture.
-FROM build-base as build-go-mod
+FROM build-base as build
+
+ARG TARGETPLATFORM
+
+# Some dependencies have to installed 
+# for the target platform: https://github.com/tonistiigi/xx#go--cgo
+RUN xx-apk add musl-dev gcc clang lld
 
 # Configure workspace
 WORKDIR /workspace
@@ -37,21 +38,6 @@ COPY go.sum go.sum
 # Cache modules
 RUN go mod download
 
-
-# Build stage install per target platform
-# dependency and effectively cross compile the application.
-FROM build-go-mod as build
-
-ARG TARGETPLATFORM
-
-COPY --from=libgit2-libs /usr/local/ /usr/local/
-
-# Some dependencies have to installed 
-# for the target platform: https://github.com/tonistiigi/xx#go--cgo
-RUN xx-apk add musl-dev gcc clang lld
-
-WORKDIR /workspace
-
 # Copy source code
 COPY main.go main.go
 COPY controllers/ controllers/
@@ -60,11 +46,13 @@ COPY internal/ internal/
 
 ARG TARGETPLATFORM
 ARG TARGETARCH
+
+# Reasons why CGO is in use:
+# - The SHA1 implementation (sha1cd) used by go-git depends on CGO for
+#   performance reasons. See: https://github.com/pjbgf/sha1cd/issues/15
 ENV CGO_ENABLED=1
 
-# Instead of using xx-go, (cross) compile with vanilla go leveraging musl tool chain.
-RUN export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig" && \
-  export CGO_LDFLAGS="$(pkg-config --static --libs --cflags libgit2) -static -fuse-ld=lld" && \
+RUN export CGO_LDFLAGS="-static -fuse-ld=lld" && \
   xx-go build \
   -ldflags "-s -w" \
   -tags 'netgo,osusergo,static_build' \
@@ -73,7 +61,6 @@ RUN export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig" && \
 # Ensure that the binary was cross-compiled correctly to the target platform.
 RUN xx-verify --static /source-controller
 
-
 FROM alpine:3.16
 
 ARG TARGETPLATFORM
@@ -82,7 +69,6 @@ RUN apk --no-cache add ca-certificates \
 
 # Copy over binary from build
 COPY --from=build /source-controller /usr/local/bin/
-COPY ATTRIBUTIONS.md /
 
 USER 65534:65534
 ENTRYPOINT [ "source-controller" ]

Dockerfile

@@ -2,10 +2,6 @@
 IMG ?= fluxcd/source-controller
 TAG ?= latest
 
-# Base image used to build the Go binary
-LIBGIT2_IMG ?= ghcr.io/fluxcd/golang-with-libgit2-only
-LIBGIT2_TAG ?= v0.4.0
-
 # Allows for defining additional Go test args, e.g. '-tags integration'.
 GO_TEST_ARGS ?= -race
 
@@ -39,14 +35,6 @@ ENVTEST_BIN_VERSION ?= 1.24.0
 # each fuzzer should run for.
 FUZZ_TIME ?= 1m
 
-# Caches libgit2 versions per tag, "forcing" rebuild only when needed.
-LIBGIT2_PATH := $(BUILD_DIR)/libgit2/$(LIBGIT2_TAG)
-LIBGIT2_LIB_PATH := $(LIBGIT2_PATH)/lib
-LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.a
-
-export CGO_ENABLED=1
-export PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig
-export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libgit2 2>/dev/null)
 GO_STATIC_FLAGS=-ldflags "-s -w" -tags 'netgo,osusergo,static_build$(addprefix ,,$(GO_TAGS))'
 
 # API (doc) generation utilities
@@ -75,11 +63,11 @@ endif
 
 all: build
 
-build: check-deps $(LIBGIT2) ## Build manager binary
+build: check-deps ## Build manager binary
 	go build $(GO_STATIC_FLAGS) -o $(BUILD_DIR)/bin/manager main.go
 
 KUBEBUILDER_ASSETS?="$(shell $(ENVTEST) --arch=$(ENVTEST_ARCH) use -i $(ENVTEST_KUBERNETES_VERSION) --bin-dir=$(ENVTEST_ASSETS_DIR) -p path)"
-test: $(LIBGIT2) install-envtest test-api check-deps ## Run all tests
+test: install-envtest test-api check-deps ## Run all tests
 	HTTPS_PROXY="" HTTP_PROXY="" \
 	KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) \
 	GIT_CONFIG_GLOBAL=/dev/null \
@@ -88,7 +76,7 @@ test: $(LIBGIT2) install-envtest test-api check-deps ## Run all tests
 	  $(GO_TEST_ARGS) \
 	  -coverprofile cover.out
 
-test-ctrl: $(LIBGIT2) install-envtest test-api check-deps ## Run controller tests
+test-ctrl: install-envtest test-api check-deps ## Run controller tests
 	HTTPS_PROXY="" HTTP_PROXY="" \
 	KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) \
 	GIT_CONFIG_GLOBAL=/dev/null \
@@ -105,7 +93,7 @@ endif
 test-api: ## Run api tests
 	cd api; go test $(GO_TEST_ARGS) ./... -coverprofile cover.out
 
-run: $(LIBGIT2) generate fmt vet manifests  ## Run against the configured Kubernetes cluster in ~/.kube/config
+run: generate fmt vet manifests  ## Run against the configured Kubernetes cluster in ~/.kube/config
 	go run $(GO_STATIC_FLAGS) ./main.go
 
 install: manifests  ## Install CRDs into a cluster
@@ -139,7 +127,7 @@ fmt:  ## Run go fmt against code
 	go fmt ./...
 	cd api; go fmt ./...
 
-vet: $(LIBGIT2)	## Run go vet against code
+vet:  ## Run go vet against code
 	go vet ./...
 	cd api; go vet ./...
 
@@ -148,8 +136,6 @@ generate: controller-gen  ## Generate API code
 
 docker-build:  ## Build the Docker image
 	docker buildx build \
-		--build-arg LIBGIT2_IMG=$(LIBGIT2_IMG) \
-		--build-arg LIBGIT2_TAG=$(LIBGIT2_TAG) \
 		--platform=$(BUILD_PLATFORMS) \
 		-t $(IMG):$(TAG) \
 		$(BUILD_ARGS) .
@@ -182,40 +168,14 @@ install-envtest: setup-envtest ## Download envtest binaries locally.
 # setup-envtest sets anything below k8s to 0555
 	chmod -R u+w $(BUILD_DIR)/testbin
 
-libgit2: $(LIBGIT2)  ## Detect or download libgit2 library
-
-COSIGN = $(GOBIN)/cosign
-$(LIBGIT2):
-	$(call go-install-tool,$(COSIGN),github.com/sigstore/cosign/cmd/cosign@latest)
-
-	IMG=$(LIBGIT2_IMG) TAG=$(LIBGIT2_TAG) PATH=$(PATH):$(GOBIN) ./hack/install-libraries.sh
-
-
 .PHONY: help
 help:  ## Display this help menu
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
-update-attributions:
-	./hack/update-attributions.sh
-
 e2e:
 	./hack/ci/e2e.sh
 
-verify: update-attributions fmt vet manifests api-docs tidy
-ifneq ($(shell grep -o 'LIBGIT2_IMG ?= \w.*' Makefile | cut -d ' ' -f 3):$(shell grep -o 'LIBGIT2_TAG ?= \w.*' Makefile | cut -d ' ' -f 3), \
-		$(shell grep -o "LIBGIT2_IMG=\w.*" Dockerfile | cut -d'=' -f2):$(shell grep -o "LIBGIT2_TAG=\w.*" Dockerfile | cut -d'=' -f2))
-	@{ \
-	echo "LIBGIT2_IMG and LIBGIT2_TAG must match in both Makefile and Dockerfile"; \
-	exit 1; \
-	}
-endif
-ifneq ($(shell grep -o 'LIBGIT2_TAG ?= \w.*' Makefile | cut -d ' ' -f 3), $(shell grep -o "LIBGIT2_TAG=.*" tests/fuzz/oss_fuzz_prebuild.sh | sed 's;LIBGIT2_TAG="$${LIBGIT2_TAG:-;;g' | sed 's;}";;g'))
-	@{ \
-	echo "LIBGIT2_TAG must match in both Makefile and tests/fuzz/oss_fuzz_prebuild.sh"; \
-	exit 1; \
-	}
-endif
-
+verify: fmt vet manifests api-docs tidy
 	@if [ ! "$$(git status --porcelain --untracked-files=no)" = "" ]; then \
 		echo "working directory is dirty:"; \
 		git --no-pager diff; \
@@ -236,7 +196,7 @@ rm -rf $$TMP_DIR ;\
 endef
 
 # Build fuzzers used by oss-fuzz.
-fuzz-build: $(LIBGIT2)
+fuzz-build:
 	rm -rf $(shell pwd)/build/fuzz/
 	mkdir -p $(shell pwd)/build/fuzz/out/
 
@@ -260,15 +220,3 @@ fuzz-native:
 	KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) \
 	FUZZ_TIME=$(FUZZ_TIME) \
 		./tests/fuzz/native_go_run.sh
-
-# Creates an env file that can be used to load all source-controller's dependencies
-# this is handy when you want to run adhoc debug sessions on tests or start the 
-# controller in a new debug session.
-env: $(LIBGIT2)
-	echo 'GO_ENABLED="1"' > $(BUILD_DIR)/.env
-	echo 'PKG_CONFIG_PATH="$(PKG_CONFIG_PATH)"' >> $(BUILD_DIR)/.env
-	echo 'LIBRARY_PATH="$(LIBRARY_PATH)"' >> $(BUILD_DIR)/.env
-	echo 'CGO_CFLAGS="$(CGO_CFLAGS)"' >> $(BUILD_DIR)/.env
-	echo 'CGO_LDFLAGS="$(CGO_LDFLAGS)"' >> $(BUILD_DIR)/.env
-	echo 'KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS)' >> $(BUILD_DIR)/.env
-	echo 'GIT_CONFIG_GLOBAL=/dev/null' >> $(BUILD_DIR)/.env

Makefile

@@ -97,6 +97,8 @@ type GitRepositorySpec struct {
 
 	// GitImplementation specifies which Git client library implementation to
 	// use. Defaults to 'go-git', valid values are ('go-git', 'libgit2').
+	// Deprecated: gitImplementation is deprecated now that 'go-git' is the
+	// only supported implementation.
 	// +kubebuilder:validation:Enum=go-git;libgit2
 	// +kubebuilder:default:=go-git
 	// +optional

api/v1beta2/gitrepository_types.go

@@ -411,9 +411,10 @@ spec:
                 type: object
               gitImplementation:
                 default: go-git
-                description: GitImplementation specifies which Git client library
-                  implementation to use. Defaults to 'go-git', valid values are ('go-git',
-                  'libgit2').
+                description: 'GitImplementation specifies which Git client library
+                  implementation to use. Defaults to ''go-git'', valid values are
+                  (''go-git'', ''libgit2''). Deprecated: gitImplementation is deprecated
+                  now that ''go-git'' is the only supported implementation.'
                 enum:
                 - go-git
                 - libgit2

config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml

@@ -1,24 +1,8 @@
 apiVersion: source.toolkit.fluxcd.io/v1beta1
 kind: GitRepository
 metadata:
-  name: large-repo-go-git
+  name: large-repo
 spec:
-  gitImplementation: go-git
-  interval: 10m
-  timeout: 2m
-  url: https://github.com/hashgraph/hedera-mirror-node.git
-  ref:
-    branch: main
-  ignore: |
-    /*
-    !/charts
----
-apiVersion: source.toolkit.fluxcd.io/v1beta1
-kind: GitRepository
-metadata:
-  name: large-repo-libgit2
-spec:
-  gitImplementation: libgit2
   interval: 10m
   timeout: 2m
   url: https://github.com/hashgraph/hedera-mirror-node.git