ci: add test coverage to e2e (#1684)

* ci: add test coverage to e2e

Signed-off-by: David Son <davbson@amazon.com>

* ci: complete e2e coverage with instrumented binary

Signed-off-by: ayush-panta <ayushkp@amazon.com>

---------

Signed-off-by: David Son <davbson@amazon.com>
Signed-off-by: ayush-panta <ayushkp@amazon.com>
Co-authored-by: David Son <davbson@amazon.com>

Author: ayush-panta

.github/workflows/e2e-linux.yaml

@@ -46,7 +46,7 @@ jobs:
     steps:
       - name: Clean macOS runner workspace
         run: |
-          rm -rf ${{ github.workspace }}/*
+          sudo rm -rf ${{ github.workspace }}/*
       - name: Allow Node16 on AL2
         if: ${{ (startsWith(inputs.os, 'amazon') && inputs.version == '2' ) }}
         run: |
@@ -90,10 +90,11 @@ jobs:
           sudo rm -rf /var/lib/containerd
           sudo rm -rf /var/soci-snapshotter
           sudo rm -rf ./_output
+          sudo rm -rf ./cov
           sudo rm -rf /run/containerd || true
       - name: Build project
         run: |
-          ./contrib/packaging/rpm/build.sh --local
+          E2E_COVERAGE=1 ./contrib/packaging/rpm/build.sh --local
       - name: Install Dependencies
         run: |
           sudo yum install iptables -y
@@ -133,6 +134,8 @@ jobs:
           sudo -E env "PATH=$PATH" INSTALLED=true REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} make test-e2e-container
           sudo -E env "PATH=$PATH" INSTALLED=true REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} make test-e2e-vm
           sudo -E env "PATH=/usr/libexec/finch:/usr/libexec/finch/cni/bin/:/usr/local/bin:$PATH" NERDCTL_TOML=/etc/finch/nerdctl/nerdctl.toml BUILDKIT_HOST=unix:///var/lib/finch/buildkit/buildkitd.sock  make test-e2e-daemon-linux
+      - name: Show test coverage
+        run: make test-e2e-cov
       - name: Change ownership of reports
         if: always()
         run: |

.github/workflows/e2e-macos.yaml

@@ -85,14 +85,17 @@ jobs:
       - name: Build project
         run: |
           export PATH="/opt/homebrew/opt/libtool/libexec/gnubin:$PATH"
-          make
+          E2E_COVERAGE=1 make
         shell: zsh {0}
       - name: Run e2e tests
         run: |
           git status
           git clean -f -d
           REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} make ${{ inputs.test-command }}
         shell: zsh {0}
+      - name: Show test coverage
+        if: ${{ inputs.test-command != 'test-e2e-daemon' }}
+        run: make test-e2e-cov
       - name: Set artifacts name outputs
         if: always()
         id: set-multiple-vars

.github/workflows/e2e-ubuntu-finch.yaml

@@ -79,7 +79,7 @@ jobs:
     steps:
       - name: Clean Ubuntu workspace
         run: |
-          rm -rf ${{ github.workspace }}/*
+          sudo rm -rf ${{ github.workspace }}/*
       - name: Clean existing Finch environment
         run: |
           # Stop services
@@ -115,6 +115,7 @@ jobs:
 
           # Clean up root cache
           sudo rm -rf /root/.cache || true
+          sudo rm -rf ${{ github.workspace }}/cov || true
           
           # Reload systemd after cleanup
           sudo systemctl daemon-reload
@@ -150,7 +151,7 @@ jobs:
           sudo apt install iptables -y
       - name: Build for Ubuntu ${{ inputs.output-arch }}
         run: |
-          make
+          E2E_COVERAGE=1 make
       - name: Generate deb
         run: |
           ./contrib/packaging/deb/package.sh --${{ inputs.output-arch }} --version ${{ needs.get-tag-and-version.outputs.version }}
@@ -173,8 +174,10 @@ jobs:
           git status
           git clean -f -d
           eval $(ssh-agent)
-          sudo -E env "PATH=$PATH" INSTALLED=true make test-e2e-container
-          sudo -E env "PATH=$PATH" INSTALLED=true make test-e2e-vm
+          sudo -E env "PATH=$PATH" INSTALLED=false make test-e2e-container
+          sudo -E env "PATH=$PATH" INSTALLED=false make test-e2e-vm
+      - name: Show test coverage
+        run: make test-e2e-cov
       - name: Change ownership of reports
         if: always()
         run: |

.github/workflows/e2e-windows.yaml

@@ -75,6 +75,7 @@ jobs:
         run: |
           Remove-Item C:\Users\Administrator\.finch -Recurse -ErrorAction Ignore
           Remove-Item C:\Users\Administrator\AppData\Local\.finch -Recurse -ErrorAction Ignore
+          Remove-Item ${{ github.workspace }}\cov -Recurse -ErrorAction Ignore
           make clean
           cd deps/finch-core && make clean
       - uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0
@@ -84,6 +85,7 @@ jobs:
       - name: Build project
         run: |
           git status
+          $env:E2E_COVERAGE="1"
           make
       - name: Run e2e tests
         run: |
@@ -98,6 +100,8 @@ jobs:
           git clean -f -d
 
           make ${{ inputs.test-command }}
+      - name: Show test coverage
+        run: make test-e2e-cov
       - name: Set artifacts name outputs
         if: always()
         id: set-multiple-vars

.gitignore

@@ -7,6 +7,7 @@ _vde_output/
 **/*.tar.gz
 **/*.lz4
 **/*.img
+cov/
 tmp/
 .vscode/
 tools_bin/

Makefile

@@ -8,6 +8,7 @@ DEST := $(shell echo "$(DESTDIR)/$(PREFIX)" | sed 's:///*:/:g; s://*$$::')
 BINDIR ?= /usr/local/bin
 OUTDIR ?= $(CURDIR)/_output
 OS_OUTDIR ?= $(OUTDIR)/os
+COVERAGE_DIR ?= $(CURDIR)/cov
 REPORT_DIR ?= $(CURDIR)/reports
 RUN_ID ?= $(GITHUB_RUN_ID)
 RUN_ATTEMPT ?= $(GITHUB_RUN_ATTEMPT)
@@ -174,8 +175,9 @@ finch-unix: finch-all
 finch-native: GO_BUILD_TAGS += native
 finch-native: finch-all
 
+E2E_COVERAGE ?=
 finch-all:
-	$(GO) build -ldflags $(LDFLAGS) -tags "$(GO_BUILD_TAGS)" -o $(OUTDIR)/bin/$(BINARYNAME) $(PACKAGE)/cmd/finch
+	$(GO) build $(if $(E2E_COVERAGE),-cover) -ldflags $(LDFLAGS) -tags "$(GO_BUILD_TAGS)" -o $(OUTDIR)/bin/$(BINARYNAME) $(PACKAGE)/cmd/finch
 
 .PHONY: release
 release: check-licenses all download-licenses
@@ -185,7 +187,7 @@ $(OUTDIR)/finch-daemon/finch@.service:
 
 .PHONY: coverage
 coverage:
-	go test $(shell go list ./... | grep -v e2e | grep -v benchmark | grep -v mocks) -coverprofile=test-coverage.out
+	go test $(shell go list ./... | grep -v e2e | grep -v benchmark | grep -v mocks)
 	go tool cover -html=test-coverage.out
 
 .PHONY: download-licenses
@@ -299,6 +301,10 @@ test-unit:
 # Container tests and VM tests can be run in any order, but they must be run sequentially.
 # For more details, see the package-level comment of the e2e package.
 
+.PHONY: create-coverage-dir
+create-coverage-dir:
+	mkdir -p $(COVERAGE_DIR)
+
 .PHONY: create-report-dir
 create-report-dir:
 	mkdir -p $(REPORT_DIR)
@@ -307,16 +313,20 @@ create-report-dir:
 test-e2e: test-e2e-vm-serial test-e2e-container
 
 .PHONY: test-e2e-vm-serial
-test-e2e-vm-serial: create-report-dir
-	go test -ldflags $(LDFLAGS) -timeout 2h ./e2e/vm -test.v -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-vm-serial-report.json --installed="$(INSTALLED)"
+test-e2e-vm-serial: create-report-dir create-coverage-dir
+	FINCH_GOCOVERDIR=$(COVERAGE_DIR) go test -coverpkg=./... -ldflags $(LDFLAGS) -timeout 2h ./e2e/vm -test.v -test.gocoverdir=$(COVERAGE_DIR) -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-vm-serial-report.json --installed="$(INSTALLED)"
 
 .PHONY: test-e2e-container
-test-e2e-container: create-report-dir
-	go test -ldflags $(LDFLAGS) -timeout 2h ./e2e/container -test.v -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-container-report.json --installed="$(INSTALLED)"
+test-e2e-container: create-report-dir create-coverage-dir
+	FINCH_GOCOVERDIR=$(COVERAGE_DIR) go test -coverpkg=./... -ldflags $(LDFLAGS) -timeout 2h ./e2e/container -test.v -test.gocoverdir=$(COVERAGE_DIR) -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-container-report.json --installed="$(INSTALLED)"
 
 .PHONY: test-e2e-vm
-test-e2e-vm: create-report-dir
-	go test -ldflags $(LDFLAGS) -timeout 2h ./e2e/vm -test.v -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-vm-report.json --installed="$(INSTALLED)" --registry="$(REGISTRY)"
+test-e2e-vm: create-report-dir create-coverage-dir
+	FINCH_GOCOVERDIR=$(COVERAGE_DIR) go test -coverpkg=./... -ldflags $(LDFLAGS) -timeout 2h ./e2e/vm -test.v -test.gocoverdir=$(COVERAGE_DIR) -ginkgo.v -ginkgo.timeout=2h  -ginkgo.focus "updates init-only config values when values are changed after init" -ginkgo.flake-attempts=3 -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-vm-report.json --installed="$(INSTALLED)" --registry="$(REGISTRY)"
+
+.PHONY: test-e2e-cov
+test-e2e-cov:
+	go tool covdata percent -i=$(COVERAGE_DIR) -pkg=$(shell go list ./... | grep -v e2e | grep -v benchmark | grep -v version | tr '\n' ',')
 
 GINKGO = go run github.com/onsi/ginkgo/v2/ginkgo
 # Common ginkgo options: -v for verbose mode, --focus="test name" for running single tests

coverage/coverage.go

@@ -26,6 +26,7 @@ func main() {
 	}
 
 	cmd := exec.Command("go", "tool", "cover", "-func=coverage.out")
+
 	output, err := cmd.Output()
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "Error executing coverage command:", err)

e2e/e2e.go

@@ -79,6 +79,13 @@ func CreateOption() (*option.Option, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize a testing option: %w", err)
 	}
+
+	// Explicitly pass GOCOVERDIR for coverage collection
+	// Use FINCH_GOCOVERDIR to avoid conflict with test process's GOCOVERDIR
+	if gocoverdir := os.Getenv("FINCH_GOCOVERDIR"); gocoverdir != "" {
+		o.UpdateEnv("GOCOVERDIR", gocoverdir)
+	}
+
 	return o, nil
 }