Merge pull request crossplane#6405 from negz/cowardice

Re-run flaky E2E tests up to 3 times

Author: negz

.github/workflows/ci.yml

@@ -288,7 +288,7 @@ jobs:
       - name: Run E2E Tests
         run: |
           earthly --strict --allow-privileged --remote-cache ghcr.io/crossplane/earthly-cache:${{ github.job }}-${{ matrix.test-suite}} \
-            +e2e --FLAGS="-test.failfast -fail-fast -prior-crossplane-version=${CROSSPLANE_PRIOR_VERSION} --test-suite ${{ matrix.test-suite }}"
+            +e2e --GOTESTSUM_FORMAT="testname" --FLAGS="-test.failfast -fail-fast -prior-crossplane-version=${CROSSPLANE_PRIOR_VERSION} --test-suite ${{ matrix.test-suite }}"
 
       - name: Publish E2E Test Flakes
         if: '!cancelled()'

Earthfile

@@ -51,6 +51,7 @@ e2e:
   ARG GOARCH=${TARGETARCH}
   ARG GOOS=${TARGETOS}
   ARG FLAGS="-test-suite=base"
+  ARG GOTESTSUM_FORMAT="standard-verbose"
   # Using earthly image to allow compatibility with different development environments e.g. WSL
   FROM earthly/dind:alpine-3.20-docker-26.1.5-r0
   RUN wget https://dl.google.com/go/go${GO_VERSION}.${GOOS}-${GOARCH}.tar.gz
@@ -71,7 +72,12 @@ e2e:
     WITH DOCKER --load crossplane-e2e/crossplane:latest=(+image --CROSSPLANE_VERSION=v0.0.0-e2e)
       # TODO(negz:) Set GITHUB_ACTIONS=true and use RUN --raw-output when
       # https://github.com/earthly/earthly/issues/4143 is fixed.
-      RUN gotestsum --no-color=false --format testname --junitfile e2e-tests.xml --raw-command go tool test2json -t -p E2E ./e2e -test.v ${FLAGS}
+      RUN gotestsum \
+        --rerun-fails \
+      	--no-color=false \
+	--format ${GOTESTSUM_FORMAT} \
+	--junitfile e2e-tests.xml \
+	--raw-command go tool test2json -t -p E2E ./e2e -test.v ${FLAGS}
     END
   FINALLY
     SAVE ARTIFACT --if-exists e2e-tests.xml AS LOCAL _output/tests/e2e-tests.xml

test/e2e/README.md

@@ -89,21 +89,6 @@ $ apk add kubectl
 # Now you can use kubectl to access the cluster
 ```
 
-### Test Logging
-
-The E2E tests are run using [`gotestsum`] that helps output the test results in
-a format that integrates easily with CI systems. The downside though is that
-test logging statements, like `t.Logf("foo")`, are not streamed in real time and
-they are only included in the output if there is a failure.
-
-To change this behavior when running the E2E tests locally, you can update the
-`Earthfile` to call `gotestsum` with a format like `--format standard-verbose`
-instead, for example:
-
-```Dockerfile
-RUN gotestsum --no-color=false --format standard-verbose ...
-```
-
 ### Running Tests Directly
 
 The E2E tests are typically run via `earthly` for convenience and consistency.