Merge pull request moby#4938 from tonistiigi/codecov-integration-tests

hack: track helper process coverage in integration tests

Author: tonistiigi

.github/workflows/.test.yml

@@ -163,6 +163,7 @@ jobs:
             *.output=type=docker,name=${{ env.TEST_IMAGE_ID }}
         env:
           BUILDKITD_TAGS: ${{ matrix.tags }}
+          TEST_COVERAGE: 1
       -
         name: Test
         run: |
@@ -180,6 +181,7 @@ jobs:
         with:
           directory: ./bin/testreports
           flags: ${{ matrix.codecov_flags }}
+          disable_file_fixes: false
           token: ${{ secrets.CODECOV_TOKEN }}  # used to upload coverage reports: https://github.com/moby/buildkit/pull/4660#issue-2142122533
       -
         name: Generate annotations

.github/workflows/test-os.yml

@@ -146,6 +146,7 @@ jobs:
           directory: ./bin/testreports
           env_vars: RUNNER_OS
           flags: unit
+          disable_file_fixes: false
           token: ${{ secrets.CODECOV_TOKEN }}  # used to upload coverage reports: https://github.com/moby/buildkit/pull/4660#issue-2142122533
       -
         name: Generate annotations

Dockerfile

@@ -87,11 +87,12 @@ RUN --mount=target=. \
 FROM buildkit-base AS buildctl
 ENV CGO_ENABLED=0
 ARG TARGETPLATFORM
+ARG GOBUILDFLAGS
 RUN --mount=target=. --mount=target=/root/.cache,type=cache \
   --mount=target=/go/pkg/mod,type=cache \
   --mount=source=/tmp/.ldflags,target=/tmp/.ldflags,from=buildkit-version <<EOT
   set -ex
-  xx-go build -ldflags "$(cat /tmp/.ldflags)" -o /usr/bin/buildctl ./cmd/buildctl
+  xx-go build ${GOBUILDFLAGS} -ldflags "$(cat /tmp/.ldflags)" -o /usr/bin/buildctl ./cmd/buildctl
   xx-verify --static /usr/bin/buildctl
   if [ "$(xx-info os)" = "linux" ]; then /usr/bin/buildctl --version; fi
 EOT
@@ -328,14 +329,37 @@ ARG TARGETPLATFORM
 RUN --mount=target=/root/.cache,type=cache <<EOT
   set -ex
   xx-go install "gotest.tools/gotestsum@${GOTESTSUM_VERSION}"
+  xx-go install "github.com/wadey/gocovmerge@latest"
   mkdir /out
   if ! xx-info is-cross; then
     /go/bin/gotestsum --version
     mv /go/bin/gotestsum /out
+    mv /go/bin/gocovmerge /out
   else
     mv /go/bin/*/gotestsum* /out
+    mv /go/bin/*/gocovmerge* /out
   fi
 EOT
+COPY --chmod=755 <<"EOF" /out/gotestsumandcover
+#!/bin/sh
+set -x
+if [ -z "$GO_TEST_COVERPROFILE" ]; then
+  exec gotestsum "$@"
+fi
+coverdir="$(dirname "$GO_TEST_COVERPROFILE")"
+mkdir -p "$coverdir/helpers"
+gotestsum "$@" "-coverprofile=$GO_TEST_COVERPROFILE"
+ecode=$?
+go tool covdata textfmt -i=$coverdir/helpers -o=$coverdir/helpers-report.txt
+gocovmerge "$coverdir/helpers-report.txt" "$GO_TEST_COVERPROFILE" > "$coverdir/merged-report.txt"
+mv "$coverdir/merged-report.txt" "$GO_TEST_COVERPROFILE"
+rm "$coverdir/helpers-report.txt"
+for f in "$coverdir/helpers"/*; do
+  rm "$f"
+done
+rmdir "$coverdir/helpers"
+exit $ecode
+EOF
 
 FROM buildkit-export AS buildkit-linux
 COPY --link --from=binaries / /usr/bin/

codecov.yml

@@ -10,3 +10,6 @@ coverage:
 
 github_checks:
   annotations: false
+
+ignore:
+  - "**/*.pb.go"

docker-bake.hcl

@@ -46,6 +46,11 @@ variable "GOLANGCI_LINT_MULTIPLATFORM" {
 variable "DESTDIR" {
   default = ""
 }
+
+variable "TEST_COVERAGE" {
+  default = null
+}
+
 function "bindir" {
   params = [defaultdir]
   result = DESTDIR != "" ? DESTDIR : "./bin/${defaultdir}"
@@ -115,6 +120,9 @@ target "integration-tests-base" {
 target "integration-tests" {
   inherits = ["integration-tests-base"]
   target = "integration-tests"
+  args = {
+    GOBUILDFLAGS = TEST_COVERAGE == "1" ? "-cover" : null
+  }
 }
 
 group "validate" {

hack/test

@@ -13,6 +13,7 @@ set -eu -o pipefail
 : "${VERIFYFLAGS=}"
 : "${CGO_ENABLED=}"
 
+: "${TEST_COVERAGE=}"
 : "${TEST_IMAGE_BUILD=1}"
 : "${TEST_IMAGE_ID=buildkit-tests}"
 : "${TEST_INTEGRATION=}"
@@ -69,7 +70,7 @@ testReportsDir="$(pwd)/bin/testreports"
 mkdir -p "$testReportsDir"
 testReportsVol="-v $testReportsDir:/testreports"
 gotestsumArgs="--format=standard-verbose --jsonfile=/testreports/go-test-report$TEST_REPORT_SUFFIX.json --junitfile=/testreports/junit-report$TEST_REPORT_SUFFIX.xml"
-gotestArgs="-mod=vendor -coverprofile=/testreports/coverage-report$TEST_REPORT_SUFFIX.txt -covermode=atomic"
+gotestArgs="-mod=vendor"
 
 if [[ "$GOBUILDFLAGS" == *"-race"* ]]; then
   if [ "$CGO_ENABLED" != "1" ]; then
@@ -81,6 +82,11 @@ if [[ "$GOBUILDFLAGS" == *"-race"* ]]; then
   export VERIFYFLAGS="" # prevent -static verification
 fi
 
+if [ "$TEST_COVERAGE" == "1" ]; then
+  export GOBUILDFLAGS="$GOBUILDFLAGS -cover"
+  export GO_TEST_COVERPROFILE="/testreports/coverage-report$TEST_REPORT_SUFFIX.txt"
+fi
+
 if [ "$TEST_IMAGE_BUILD" = "1" ]; then
   buildxCmd build $cacheFromFlags \
     --build-arg ALPINE_VERSION \
@@ -118,14 +124,15 @@ baseCreateFlags="--rm --privileged \
 -e GOTESTSUM_FORMAT \
 -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry \
 -e BUILDKIT_INTEGRATION_DOCKERD_FLAGS \
--e GORACE"
+-e GORACE \
+-e GO_TEST_COVERPROFILE"
 
 if [ "$TEST_INTEGRATION" == 1 ]; then
   cid=$(dockerCmd create $baseCreateFlags \
     ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} \
     -e SKIP_INTEGRATION_TESTS \
     $TEST_IMAGE_ID \
-    gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v})
+    gotestsumandcover $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v})
   if [ "$TEST_DOCKERD" = "1" ]; then
     dockerCmd cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd
   fi
@@ -170,7 +177,7 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then
           -e FRONTEND_GATEWAY_ONLY=local:/$release.tar \
           -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend \
           $TEST_IMAGE_ID \
-          gotestsum $gotestsumArgs --packages=./frontend/dockerfile -- $gotestArgs --count=1 -tags "$buildtags" ${TESTFLAGS:--v})
+          gotestsumandcover $gotestsumArgs --packages=./frontend/dockerfile -- $gotestArgs --count=1 -tags "$buildtags" ${TESTFLAGS:--v})
         dockerCmd cp $tarout $cid:/$release.tar
         if [ "$TEST_DOCKERD" = "1" ]; then
           dockerCmd cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd

util/testutil/integration/sandbox.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -65,6 +66,10 @@ func (sb *sandbox) Cmd(args ...string) *exec.Cmd {
 	cmd := exec.Command("buildctl", args...)
 	cmd.Env = append(cmd.Env, os.Environ()...)
 	cmd.Env = append(cmd.Env, "BUILDKIT_HOST="+sb.Address())
+	if v := os.Getenv("GO_TEST_COVERPROFILE"); v != "" {
+		coverDir := filepath.Join(filepath.Dir(v), "helpers")
+		cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir)
+	}
 	return cmd
 }
 

util/testutil/workers/util.go

@@ -79,6 +79,10 @@ func runBuildkitd(
 		"BUILDKIT_DEBUG_EXEC_OUTPUT=1",
 		"BUILDKIT_DEBUG_PANIC_ON_ERROR=1",
 		"TMPDIR="+filepath.Join(tmpdir, "tmp"))
+	if v := os.Getenv("GO_TEST_COVERPROFILE"); v != "" {
+		coverDir := filepath.Join(filepath.Dir(v), "helpers")
+		cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir)
+	}
 	cmd.Env = append(cmd.Env, extraEnv...)
 	cmd.SysProcAttr = getSysProcAttr()