Merge pull request #3030 from psturc/improve-acceptance-tests

feat: improve acceptance tests

Author: dheerajodha

.github/workflows/checks-codecov.yaml

@@ -91,6 +91,7 @@ jobs:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           files: ./coverage-unit.out
+          disable_search: true
           flags: unit
 
       - name: Upload generative test coverage report
@@ -99,6 +100,7 @@ jobs:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           files: ./coverage-generative.out
+          disable_search: true
           flags: generative
 
       - name: Upload integration test coverage report
@@ -107,6 +109,7 @@ jobs:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           files: ./coverage-integration.out
+          disable_search: true
           flags: integration
 
   Acceptance:
@@ -143,12 +146,16 @@ jobs:
         run: git log --oneline -n1
 
       - name: Acceptance test
-        run: make acceptance
+        id: acceptance_test
+        run: E2E_INSTRUMENTATION=true make acceptance
 
       - name: Upload coverage report
         uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           files: ./coverage-acceptance.out
+          disable_search: true
           flags: acceptance
 
   Tools:

Makefile

@@ -13,6 +13,7 @@ SHELL=$(if $@,$(info ❱ $@))$(_SHELL)
 ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 COPY:=The Conforma Contributors
 COSIGN_VERSION=$(shell go list -f '{{.Version}}' -m github.com/sigstore/cosign/v2)
+E2E_INSTRUMENTATION_FLAGS := $(if $(filter $(E2E_INSTRUMENTATION),true),-cover -covermode atomic)
 
 ##@ Information
 
@@ -54,7 +55,7 @@ BUILD_BIN_SUFFIX=$(if $(DEBUG_BUILD),_debug,)
 $(ALL_SUPPORTED_OS_ARCH): generate ## Build binaries for specific platform/architecture, e.g. make dist/ec_linux_amd64
 	@GOOS=$(word 2,$(subst _, ,$(notdir $@))); \
 	GOARCH=$(word 3,$(subst _, ,$(notdir $@))); \
-	GOOS=$${GOOS} GOARCH=$${GOARCH} CGO_ENABLED=0 go build $(BUILD_TRIMPATH) $(BUILD_GC_FLAGS) -ldflags="$(BUILD_LD_FLAGS) -X github.com/conforma/cli/internal/version.Version=$(VERSION)" -o dist/ec_$${GOOS}_$${GOARCH}$(BUILD_BIN_SUFFIX); \
+	GOOS=$${GOOS} GOARCH=$${GOARCH} CGO_ENABLED=0 go build $(E2E_INSTRUMENTATION_FLAGS) $(BUILD_TRIMPATH) $(BUILD_GC_FLAGS) -ldflags="$(BUILD_LD_FLAGS) -X github.com/conforma/cli/internal/version.Version=$(VERSION)" -o dist/ec_$${GOOS}_$${GOARCH}$(BUILD_BIN_SUFFIX); \
 	sha256sum -b dist/ec_$${GOOS}_$${GOARCH}$(BUILD_BIN_SUFFIX) > dist/ec_$${GOOS}_$${GOARCH}$(BUILD_BIN_SUFFIX).sha256
 
 .PHONY: dist
@@ -120,15 +121,13 @@ acceptance: ## Run all acceptance tests
 	cleanup() { \
 		cp "$${ACCEPTANCE_WORKDIR}"/features/__snapshots__/* "$(ROOT_DIR)"/features/__snapshots__/; \
 	}; \
+	mkdir -p "$${ACCEPTANCE_WORKDIR}/coverage"; \
 	trap cleanup EXIT; \
 	cp -R . "$$ACCEPTANCE_WORKDIR"; \
 	cd "$$ACCEPTANCE_WORKDIR" && \
-	go run acceptance/coverage/coverage.go && \
 	$(MAKE) build && \
-	export COVERAGE_FILEPATH="$$ACCEPTANCE_WORKDIR"; \
-	export COVERAGE_FILENAME="-acceptance"; \
-	cd acceptance && go test -coverprofile "$$ACCEPTANCE_WORKDIR/coverage-acceptance.out" -timeout $(ACCEPTANCE_TIMEOUT) ./... && \
-	go run -modfile "$$ACCEPTANCE_WORKDIR/tools/go.mod" github.com/wadey/gocovmerge "$$ACCEPTANCE_WORKDIR/coverage-acceptance.out" > "$(ROOT_DIR)/coverage-acceptance.out"
+	export GOCOVERDIR="$${ACCEPTANCE_WORKDIR}/coverage"; \
+	cd acceptance && go test -timeout $(ACCEPTANCE_TIMEOUT) ./... ; go tool covdata textfmt -i=$${GOCOVERDIR} -o="$(ROOT_DIR)/coverage-acceptance.out"
 
 # Add @focus above the feature you're hacking on to use this
 # (Mainly for use with the feature-% target below)

README.md

@@ -18,9 +18,14 @@ run `make dist` to build for all supported architectures.
 
 ## Testing
 
-Run `make test` to run the unit tests, and `make acceptance` to run the
+Run `make test` to run the unit tests, and `E2E_INSTRUMENTATION=true make acceptance` to run the
 acceptance tests.
 
+The purpose of the `E2E_INSTRUMENTATION=true` environment variable is to pass
+extra flags to the build command. This instruments the resulting binary,
+allowing for the collection of coverage data during the acceptance test run.
+(more information here: https://go.dev/doc/build-cover).
+
 ## Linting
 
 Run `make lint` to check for linting issues, and `make lint-fix` to fix linting

acceptance/README.md

@@ -14,7 +14,7 @@ uses the established Go test to launch Godog.
 
 To run the acceptance tests either run:
 
-    $ make acceptance
+    $ E2E_INSTRUMENTATION=true make acceptance
 
 from the root of the repository.
 
@@ -64,7 +64,7 @@ Docker website how to accomplish that.
 ## Debugging
 
 The acceptance tests execute the `ec` binary during test execution. (For this
-reason `make acceptance` builds the binary prior to running the tests.)
+reason `E2E_INSTRUMENTATION=true make acceptance` builds the binary prior to running the tests.)
 
 To use a debugger, like [delve](https://github.com/go-delve/delve), you must
 determine what part of the code is being debugged. If it's part of the

acceptance/cli/cli.go

@@ -87,8 +87,7 @@ func variables(ctx context.Context) (context.Context, []string, map[string]strin
 	// fail if it can't locate the git command
 	environment := []string{
 		"PATH=" + os.Getenv("PATH"),
-		"COVERAGE_FILEPATH=" + os.Getenv("COVERAGE_FILEPATH"), // where to put the coverage file, $COVERAGE_FILEPATH is provided by the Makefile, if empty it'll be $TMPDIR
-		"COVERAGE_FILENAME=" + os.Getenv("COVERAGE_FILENAME"), // suffix for the coverage file
+		"GOCOVERDIR=" + os.Getenv("GOCOVERDIR"), // directory where the Go coverage raw data is stored
 		"HOME=/tmp",
 	}