Implement test coverage

Refs:
 - stolostron/backlog#24298

Co-authored-by: Dale Haiducek <[email protected]>
Signed-off-by: Justin Kulikauskas <[email protected]>

Author: 2 people

.github/workflows/kind.yml

@@ -17,6 +17,7 @@ jobs:
   kind-tests:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         # KinD tags: https://hub.docker.com/r/kindest/node/tags
         # (OCP 4.6 runs Kubernetes v1.19)
@@ -51,7 +52,7 @@ jobs:
       run: |
         export GOPATH=$(go env GOPATH)
         make wait-for-work-agent
-        make e2e-test
+        make e2e-test-coverage
 
     - if: matrix.hosted_mode == 'true'
       name: E2E tests hosted mode
@@ -60,6 +61,14 @@ jobs:
         make wait-for-work-agent
         make e2e-test-hosted-mode
 
+    - name: Test Coverage Verification
+      if: |
+        github.event_name == 'pull_request' &&
+        matrix.hosted_mode == 'false'
+      run: |
+        make test-coverage
+        make coverage-verify
+
     - name: Clean up clusters
       run: |
         RUN_MODE="delete" ./build/manage-clusters.sh

Makefile

@@ -82,7 +82,11 @@ vet: ## Run go vet against code.
 
 .PHONY: test
 test: manifests generate fmt vet envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test `go list ./... | grep -v test/e2e` -coverprofile coverage.out
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test $(TESTARGS) `go list ./... | grep -v test/e2e`
+
+.PHONY: test-coverage
+test-coverage: TESTARGS = -json -cover -covermode=atomic -coverprofile=coverage_unit.out
+test-coverage: test
 
 GOSEC = $(LOCAL_BIN)/gosec
 GOSEC_VERSION = 2.9.6
@@ -159,6 +163,7 @@ HUB_KUBECONFIG ?= $(PWD)/policy-addon-ctrl1.kubeconfig
 HUB_KUBECONFIG_INTERNAL ?= $(PWD)/policy-addon-ctrl1.kubeconfig-internal
 HUB_CLUSTER_NAME ?= policy-addon-ctrl1
 MANAGED_CLUSTER_NAME ?= cluster1
+KIND_VERSION ?= latest
 ifneq ($(KIND_VERSION), latest)
 	KIND_ARGS = --image kindest/node:$(KIND_VERSION)
 else
@@ -220,7 +225,7 @@ CONTROLLER_NAMESPACE ?= governance-policy-addon-controller-system
 
 .PHONY: kind-run-local
 kind-run-local: manifests generate fmt vet $(KIND_KUBECONFIG) ## Run the policy-addon-controller locally against the kind cluster.
-	KUBECONFIG=$(KIND_KUBECONFIG) kubectl get ns $(CONTROLLER_NAMESPACE); if [ $$? -ne 0 ] ; then kubectl create ns $(CONTROLLER_NAMESPACE); fi 
+	-KUBECONFIG=$(KIND_KUBECONFIG) kubectl create ns $(CONTROLLER_NAMESPACE)
 	go run ./main.go controller --kubeconfig=$(KIND_KUBECONFIG) --namespace $(CONTROLLER_NAMESPACE)
 
 .PHONY: kind-load-image
@@ -235,24 +240,52 @@ kind-regenerate-controller: manifests generate kustomize $(KIND_KUBECONFIG) ## R
 	mv config/default/kustomization.yaml.tmp config/default/kustomization.yaml
 	KUBECONFIG=$(KIND_KUBECONFIG) kubectl delete -n $(CONTROLLER_NAMESPACE) pods -l=app=governance-policy-addon-controller
 
-DEPLOYMENT_TARGETS := kind-deploy-registration-operator-hub kind-deploy-registration-operator-managed \
-											kind-approve-cluster kind-load-image kind-regenerate-controller
+OCM_PREP_TARGETS := kind-deploy-registration-operator-hub kind-deploy-registration-operator-managed kind-approve-cluster
+.PHONY: kind-prep-ocm
+kind-prep-ocm: $(OCM_PREP_TARGETS) ## Install OCM registration pieces and connect the clusters
+
 .PHONY: kind-deploy-controller
-kind-deploy-controller: $(DEPLOYMENT_TARGETS) ## Deploy the policy-addon-controller to the kind cluster.
+kind-deploy-controller: kind-prep-ocm kind-load-image kind-regenerate-controller ## Deploy the policy-addon-controller to the kind cluster.
 
 GINKGO = $(LOCAL_BIN)/ginkgo
 .PHONY: e2e-dependencies
 e2e-dependencies: ## Download ginkgo locally if necessary.
 	$(call go-get-tool,github.com/onsi/ginkgo/v2/ginkgo@$(shell awk '/github.com\/onsi\/ginkgo\/v2/ {print $$2}' go.mod))
 
 .PHONY: e2e-test
-e2e-test: e2e-dependencies
-	$(GINKGO) -v --label-filter="!hosted-mode" --fail-fast --slow-spec-threshold=10s test/e2e
+e2e-test: e2e-dependencies ## Run E2E tests.
+	$(GINKGO) -v --label-filter="!hosted-mode" --fail-fast --slow-spec-threshold=10s $(E2E_TEST_ARGS) test/e2e
 
 .PHONY: e2e-test-hosted-mode
 e2e-test-hosted-mode: e2e-dependencies
 	$(GINKGO) -v --label-filter="hosted-mode" --fail-fast --slow-spec-threshold=10s test/e2e
 
+.PHONY: e2e-test-coverage
+e2e-test-coverage: E2E_TEST_ARGS = --json-report=report_e2e.json --output-dir=.
+e2e-test-coverage: e2e-run-instrumented e2e-test e2e-stop-instrumented ## Run E2E tests using instrumented controller.
+
+.PHONY: e2e-build-instrumented
+e2e-build-instrumented:
+	go test -covermode=atomic -coverpkg=$(shell cat go.mod | head -1 | cut -d ' ' -f 2)/... -c -tags e2e ./ -o build/_output/bin/$(IMG)-instrumented
+
+COVERAGE_E2E_OUT ?= coverage_e2e.out
+.PHONY: e2e-run-instrumented
+e2e-run-instrumented: e2e-build-instrumented
+	-KUBECONFIG=$(KIND_KUBECONFIG) kubectl create ns $(CONTROLLER_NAMESPACE)
+	CONFIG_POLICY_CONTROLLER_IMAGE="$(REGISTRY)/config-policy-controller:$(TAG)" \
+	  KUBE_RBAC_PROXY_IMAGE="registry.redhat.io/openshift4/ose-kube-rbac-proxy:v4.10" \
+	  GOVERNANCE_POLICY_SPEC_SYNC_IMAGE="$(REGISTRY)/governance-policy-spec-sync:$(TAG)" \
+	  GOVERNANCE_POLICY_STATUS_SYNC_IMAGE="$(REGISTRY)/governance-policy-status-sync:$(TAG)" \
+	  GOVERNANCE_POLICY_TEMPLATE_SYNC_IMAGE="$(REGISTRY)/governance-policy-template-sync:$(TAG)" \
+	  ./build/_output/bin/$(IMG)-instrumented -test.v -test.run="^TestRunMain$$" -test.coverprofile=$(COVERAGE_E2E_OUT) \
+	  --kubeconfig="$(KIND_KUBECONFIG)" &>build/_output/controller.log &
+
+.PHONY: e2e-stop-instrumented
+e2e-stop-instrumented:
+	ps -ef | grep '$(IMG)' | grep -v grep | awk '{print $$2}' | xargs kill -s SIGUSR1
+	sleep 5 # wait for tests to gracefully shut down
+	-ps -ef | grep '$(IMG)' | grep -v grep | awk '{print $$2}' | xargs kill
+
 .PHONY: e2e-debug
 e2e-debug: ## Collect debug logs from deployed clusters.
 	@echo "##### Gathering information from $(KIND_NAME) #####"
@@ -262,7 +295,9 @@ e2e-debug: ## Collect debug logs from deployed clusters.
 	-KUBECONFIG=$(KIND_KUBECONFIG) kubectl -n open-cluster-management-agent-addon get deployments
 	-KUBECONFIG=$(KIND_KUBECONFIG) kubectl -n open-cluster-management-agent-addon get pods
 	-KUBECONFIG=$(KIND_KUBECONFIG) kubectl get manifestwork --all-namespaces -o yaml
-
+	
+	@echo "* Local controller log:""
+	-cat build/_output/controller.log
 	@echo "* Container logs in namespace $(CONTROLLER_NAMESPACE):"
 	-@for POD in $(shell KUBECONFIG=$(KIND_KUBECONFIG) kubectl -n $(CONTROLLER_NAMESPACE) get pods -o name); do \
 		for CONTAINER in $$(KUBECONFIG=$(KIND_KUBECONFIG) kubectl -n $(CONTROLLER_NAMESPACE) get $${POD} -o jsonpath={.spec.containers[*].name}); do \
@@ -301,3 +336,19 @@ lint-dependencies:
 #    eg: lint: lint-go lint-yaml
 .PHONY: lint
 lint: lint-dependencies lint-all ## Run linting against the code.
+
+##@ Test Coverage
+GOCOVMERGE = $(LOCAL_BIN)/gocovmerge
+.PHONY: coverage-dependencies
+coverage-dependencies:
+	$(call go-get-tool,github.com/wadey/[email protected])
+
+COVERAGE_FILE = coverage.out
+.PHONY: coverage-merge
+coverage-merge: coverage-dependencies ## Merge coverage reports.
+	@echo Merging the coverage reports into $(COVERAGE_FILE)
+	$(GOCOVMERGE) $(PWD)/coverage_* > $(COVERAGE_FILE)
+
+.PHONY: coverage-verify
+coverage-verify: ## Verify coverage percentage meets coverage thresholds.
+	./build/common/scripts/coverage_calc.sh

build/manage-clusters.sh

@@ -1,6 +1,10 @@
 #! /bin/bash
 
-RUN_MODE=${RUN_MODE:-"create"}
+if [[ "${HOSTED_MODE}" == "true" ]]; then
+  RUN_MODE=${RUN_MODE:-"create"}
+else
+  RUN_MODE=${RUN_MODE:-"create-dev"}
+fi
 
 # Number of managed clusters
 MANAGED_CLUSTER_COUNT=${MANAGED_CLUSTER_COUNT:-1}
@@ -32,6 +36,9 @@ case ${RUN_MODE} in
   create)
     make kind-deploy-controller
     ;;
+  create-dev)
+    make kind-prep-ocm
+    ;;
 esac
 
 # Deploy a variable number of managed clusters starting with cluster2
@@ -45,7 +52,7 @@ for i in $(seq 2 $((MANAGED_CLUSTER_COUNT+1))); do
     debug)
       make e2e-debug
       ;;
-    create)
+    create | create-dev)
       if [[ "${HOSTED_MODE}" == "true" ]]; then
         make kind-deploy-registration-operator-managed-hosted
       else

main_test.go

@@ -0,0 +1,33 @@
+//go:build e2e
+// +build e2e
+
+// Copyright (c) 2022 Red Hat, Inc.
+// Copyright Contributors to the Open Cluster Management project
+
+package main
+
+import (
+	"os"
+	"os/signal"
+	"syscall"
+	"testing"
+)
+
+// TestRunMain wraps the main() function in order to build a test binary and collection coverage for
+// E2E/Integration tests. Controller CLI flags are also passed in here.
+func TestRunMain(t *testing.T) {
+	// Modifying the args seems necessary to make the test binary and cobra happy.
+	os.Args = append([]string{
+		"governance-policy-addon-controller",
+		"controller",
+		"--namespace=governance-policy-addon-controller-system",
+	}, os.Args[1:]...)
+
+	// Run main in a separate goroutine
+	go main()
+
+	// The test will end (and pass) via an external signal
+	stopHandler := make(chan os.Signal, 1)
+	signal.Notify(stopHandler, syscall.SIGUSR1)
+	<-stopHandler
+}