Merge pull request moby#3727 from crazy-max/test-annotations

ci: output annotations for failures

Author: crazy-max

.github/workflows/.test.yml

@@ -31,6 +31,7 @@ env:
   SETUP_BUILDX_VERSION: "latest"
   SETUP_BUILDKIT_IMAGE: "moby/buildkit:latest"
   TESTFLAGS: "-v --parallel=6 --timeout=30m"
+  GOTESTSUM_FORMAT: "standard-verbose"
 
 jobs:
   prepare:
@@ -143,6 +144,7 @@ jobs:
         name: Test
         continue-on-error: ${{ matrix.tags == 'nydus' }}
         run: |
+          export TEST_REPORT_SUFFIX=-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.kind }}-${{ matrix.worker }}-${{ matrix.tags }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]')
           if [ -n "${{ matrix.tags }}" ]; then
             TESTFLAGS="${TESTFLAGS} --tags=${{ matrix.tags }}"
             export BUILDKITD_TAGS="${{ matrix.tags }}"
@@ -151,48 +153,28 @@ jobs:
             export TESTFLAGS="${TESTFLAGS} --run=//worker=${{ matrix.worker }}$"
           fi
           ./hack/test ${{ matrix.kind }}
-          mv ./coverage/coverage.txt ./coverage/coverage-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.kind }}-${{ matrix.worker }}-${{ matrix.tags }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]').txt
         env:
           TEST_COVERAGE: 1
           TESTPKGS: ${{ matrix.pkg }}
           SKIP_INTEGRATION_TESTS: ${{ matrix.skip-integration-tests }}
           CACHE_FROM: type=gha,scope=${{ inputs.cache_scope }}
       -
-        name: Upload coverage file
-        continue-on-error: ${{ matrix.tags == 'nydus' }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: coverage
-          path: ./coverage
-
-  upload-coverage:
-    runs-on: ubuntu-20.04
-    needs:
-      - run
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      -
-        name: Download coverage files
-        uses: actions/download-artifact@v3
+        name: Send to Codecov
+        if: always()
+        uses: codecov/codecov-action@v3
         with:
-          name: coverage
-          path: ./coverage
+          directory: ./bin/testreports
+          flags: ${{ matrix.codecov_flags }}
       -
-        name: List coverage files
-        id: files
-        uses: actions/github-script@v6
+        name: Generate annotations
+        if: always()
+        uses: crazy-max/.github/.github/actions/gotest-annotations@5af0882e0496d2f7e98a53ae4048e3d86682496f
         with:
-          result-encoding: string
-          script: |
-            return require('fs').readdirSync('./coverage', {withFileTypes: true})
-              .filter(item => !item.isDirectory())
-              .map(item => `./coverage/${item.name}`)
-              .join(',');
+          directory: ./bin/testreports
       -
-        name: Send to Codecov
-        uses: codecov/codecov-action@v3
+        name: Upload test reports
+        if: always()
+        uses: actions/upload-artifact@v3
         with:
-          files: ${{ steps.files.outputs.result }}
-          flags: ${{ matrix.codecov_flags }}
+          name: test-reports
+          path: ./bin/testreports

.github/workflows/test-os.yml

@@ -21,6 +21,8 @@ on:
 env:
   GO_VERSION: "1.20"
   TESTFLAGS: "-v --parallel=6 --timeout=30m"
+  GOTESTSUM_VERSION: "v1.9.0"  # same as one in Dockerfile
+  GOTESTSUM_FORMAT: "standard-verbose"
 
 jobs:
   test:
@@ -43,22 +45,43 @@ jobs:
           go-version: "${{ env.GO_VERSION }}"
           cache: true
       -
-        name: Go mod
+        name: Install gotestsum
         run: |
-          go mod download
+          go install gotest.tools/gotestsum@${{ env.GOTESTSUM_VERSION }}
       -
         name: Test
         env:
           TMPDIR: ${{ runner.temp }}
           SKIP_INTEGRATION_TESTS: 1
         run: |
-          mkdir -p ./coverage
-          go test -coverprofile=./coverage/coverage.txt -covermode=atomic ${TESTFLAGS} ./...
+          mkdir -p ./bin/testreports
+          gotestsum \
+            --jsonfile="./bin/testreports/go-test-report.json" \
+            --junitfile="./bin/testreports/junit-report.xml" \
+            --packages="./..." \
+            -- \
+              "-mod=vendor" \
+              "-coverprofile" "./bin/testreports/coverage.txt" \
+              "-covermode" "atomic" ${TESTFLAGS}
         shell: bash
       -
         name: Send to Codecov
+        if: always()
         uses: codecov/codecov-action@v3
         with:
-          files: ./coverage/coverage.txt
+          directory: ./bin/testreports
           env_vars: RUNNER_OS
           flags: unit
+      -
+        name: Generate annotations
+        if: always()
+        uses: crazy-max/.github/.github/actions/gotest-annotations@5af0882e0496d2f7e98a53ae4048e3d86682496f
+        with:
+          directory: ./bin/testreports
+      -
+        name: Upload test reports
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-reports
+          path: ./bin/testreports

.gitignore

@@ -2,6 +2,5 @@
 #  if you want to ignore files created by your editor/tools,
 #  please consider a global .gitignore https://help.github.com/articles/ignoring-files
 bin
-coverage
 .certs
 .tmp

Dockerfile

@@ -14,6 +14,7 @@ ARG NYDUS_VERSION=v2.1.0
 ARG MINIO_VERSION=RELEASE.2022-05-03T20-36-08Z
 ARG MINIO_MC_VERSION=RELEASE.2022-05-04T06-07-55Z
 ARG AZURITE_VERSION=3.18.0
+ARG GOTESTSUM_VERSION=v1.9.0
 
 ARG GO_VERSION=1.20
 ARG ALPINE_VERSION=3.17
@@ -207,6 +208,12 @@ SHELL ["/bin/bash", "-c"]
 RUN wget https://github.com/dragonflyoss/image-service/releases/download/$NYDUS_VERSION/nydus-static-$NYDUS_VERSION-$TARGETOS-$TARGETARCH.tgz
 RUN mkdir -p /out/nydus-static && tar xzvf nydus-static-$NYDUS_VERSION-$TARGETOS-$TARGETARCH.tgz -C /out
 
+FROM gobuild-base AS gotestsum
+ARG GOTESTSUM_VERSION
+RUN --mount=target=/root/.cache,type=cache \
+  GOBIN=/out/ go install "gotest.tools/gotestsum@${GOTESTSUM_VERSION}" && \
+  /out/gotestsum --version
+
 FROM buildkit-export AS buildkit-linux
 COPY --link --from=binaries / /usr/bin/
 ENTRYPOINT ["buildkitd"]
@@ -249,6 +256,8 @@ ENTRYPOINT ["/docker-entrypoint.sh"]
 ENV BUILDKIT_INTEGRATION_CONTAINERD_EXTRA="containerd-1.6=/opt/containerd-alt-16/bin"
 ENV BUILDKIT_INTEGRATION_SNAPSHOTTER=stargz
 ENV CGO_ENABLED=0
+ENV GOTESTSUM_FORMAT=standard-verbose
+COPY --link --from=gotestsum /out/gotestsum /usr/bin/
 COPY --link --from=minio /opt/bin/minio /usr/bin/
 COPY --link --from=minio-mc /usr/bin/mc /usr/bin/
 COPY --link --from=nydus /out/nydus-static/* /usr/bin/

hack/test

@@ -11,7 +11,7 @@ set -eu -o pipefail
 : ${TEST_DOCKERFILE=}
 : ${TEST_DOCKERD=}
 : ${TEST_DOCKERD_BINARY=$(which dockerd)}
-: ${TEST_COVERAGE=}
+: ${TEST_REPORT_SUFFIX=}
 : ${TEST_KEEP_CACHE=}
 : ${DOCKERFILE_RELEASES=}
 : ${BUILDKIT_WORKER_RANDOM=}
@@ -54,14 +54,11 @@ done
 iid="buildkit-tests"
 iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
 
-coverageVol=""
-coverageFlags=""
-if [ "$TEST_COVERAGE" = "1" ]; then
-  covdir="$(pwd)/coverage"
-  mkdir -p "$covdir"
-  coverageVol="-v $covdir:/coverage"
-  coverageFlags="-coverprofile=/coverage/coverage.txt -covermode=atomic"
-fi
+testReportsDir="$(pwd)/bin/testreports"
+mkdir -p "$testReportsDir"
+testReportsVol="-v $testReportsDir:/testreports"
+gotestsumArgs="--format=standard-verbose --jsonfile=/testreports/go-test-report$TEST_REPORT_SUFFIX.json --junitfile=/testreports/junit-report$TEST_REPORT_SUFFIX.xml"
+gotestArgs="-mod=vendor -coverprofile=/testreports/coverage-report$TEST_REPORT_SUFFIX.txt -covermode=atomic"
 
 buildxCmd build $cacheFromFlags \
   --build-arg BUILDKITD_TAGS \
@@ -78,7 +75,7 @@ if ! docker container inspect "$cacheVolume" >/dev/null 2>/dev/null; then
 fi
 
 if [ "$TEST_INTEGRATION" == 1 ]; then
-  cid=$(docker create --rm -v /tmp $coverageVol --volumes-from=$cacheVolume -e GITHUB_REF -e ACTIONS_RUNTIME_TOKEN -e ACTIONS_CACHE_URL -e TEST_DOCKERD -e SKIP_INTEGRATION_TESTS -e BUILDKIT_TEST_ENABLE_FEATURES -e BUILDKIT_TEST_DISABLE_FEATURES ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_INTEGRATION_DOCKERD_FLAGS --privileged $iid go test $coverageFlags ${TESTFLAGS:--v} ${TESTPKGS:-./...})
+  cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e GITHUB_REF -e ACTIONS_RUNTIME_TOKEN -e ACTIONS_CACHE_URL -e TEST_DOCKERD -e SKIP_INTEGRATION_TESTS -e BUILDKIT_TEST_ENABLE_FEATURES -e GOTESTSUM_FORMAT ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_INTEGRATION_DOCKERD_FLAGS --privileged $iid gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v})
   if [ "$TEST_DOCKERD" = "1" ]; then
     docker cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd
   fi
@@ -118,7 +115,7 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then
 
     if [ -s $tarout ]; then
       if [ "$release" = "mainline" ] || [ "$release" = "labs" ] || [ -n "$DOCKERFILE_RELEASES_CUSTOM" ] || [ "$GITHUB_ACTIONS" = "true" ]; then
-        cid=$(docker create -v /tmp $coverageVol --rm --privileged --volumes-from=$cacheVolume -e GITHUB_REF -e ACTIONS_RUNTIME_TOKEN -e ACTIONS_CACHE_URL -e TEST_DOCKERD -e BUILDKIT_TEST_ENABLE_FEATURES -e BUILDKIT_TEST_DISABLE_FEATURES -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_WORKER_RANDOM -e BUILDKIT_INTEGRATION_DOCKERD_FLAGS -e FRONTEND_GATEWAY_ONLY=local:/$release.tar -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend $iid go test $coverageFlags --count=1 -tags "$buildtags" ${TESTFLAGS:--v} ./frontend/dockerfile)
+        cid=$(docker create -v /tmp $testReportsVol --rm --privileged --volumes-from=$cacheVolume -e GITHUB_REF -e ACTIONS_RUNTIME_TOKEN -e ACTIONS_CACHE_URL -e TEST_DOCKERD -e BUILDKIT_TEST_ENABLE_FEATURES -e GOTESTSUM_FORMAT -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_WORKER_RANDOM -e BUILDKIT_INTEGRATION_DOCKERD_FLAGS -e FRONTEND_GATEWAY_ONLY=local:/$release.tar -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend $iid gotestsum $gotestsumArgs --packages=./frontend/dockerfile -- $gotestArgs --count=1 -tags "$buildtags" ${TESTFLAGS:--v})
         docker cp $tarout $cid:/$release.tar
         if [ "$TEST_DOCKERD" = "1" ]; then
           docker cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd