Enable dynamically skipping tests in JUnit XML reports (#7925)

* Add an example failing test

* Rely on TAB to determine if tests should block merge

* Revert "Add an example failing test"

This reverts commit 4d484a5.

Author: jacebrowning

.github/ci-cd-scripts/upload-results.sh

@@ -2,15 +2,17 @@
 set -euo pipefail
 
 if [ -z "${TAB_API_URL:-}" ] || [ -z "${TAB_API_KEY:-}" ]; then
+    echo "WARNING: TAB_API_URL and TAB_API_KEY must be set to analyze results"
+    grep --quiet 'failures="0"' test-results/junit.xml
     exit 0
 fi
 
-project="https://github.com/KittyCAD/modeling-app"
+project="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}"
 suite="${CI_SUITE:-unit}"
 branch="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME:-}}"
 commit="${CI_COMMIT_SHA:-${GITHUB_SHA:-}}"
 
-echo "Uploading batch results"
+echo "Uploading batch results:"
 curl --silent --request POST \
   --header "X-API-Key: ${TAB_API_KEY}" \
   --form "project=${project}" \
@@ -29,10 +31,11 @@ curl --silent --request POST \
   --form "GITHUB_SHA=${GITHUB_SHA:-}" \
   --form "GITHUB_WORKFLOW=${GITHUB_WORKFLOW:-}" \
   --form "RUNNER_ARCH=${RUNNER_ARCH:-}" \
-  ${TAB_API_URL}/api/results/bulk
+  ${TAB_API_URL}/api/results/bulk > test-results/tab.json
+cat test-results/tab.json
 
 echo
-echo "Sharing updated report"
+echo "Sharing updated report:"
 curl --silent --request POST \
   --header "Content-Type: application/json" \
   --header "X-API-Key: ${TAB_API_KEY}" \
@@ -42,3 +45,6 @@ curl --silent --request POST \
     \"commit\": \"${commit}\"
   }" \
   ${TAB_API_URL}/api/share
+echo
+
+grep --quiet '"block": false' test-results/tab.json

.github/workflows/cargo-test.yml

@@ -87,9 +87,9 @@ jobs:
           set -euo pipefail
           cd rust
           cargo nextest run \
-          --workspace --features=artifact-graph --retries=10 --no-fail-fast --profile=ci \
-          simulation_tests::kcl_samples \
-          2>&1 | tee /tmp/github-actions.log
+            --workspace --features=artifact-graph --retries=10 --no-fail-fast --profile=ci \
+            simulation_tests::kcl_samples \
+            2>&1 | tee /tmp/github-actions.log
         env:
           KITTYCAD_API_TOKEN: ${{secrets.KITTYCAD_API_TOKEN_DEV}}
           ZOO_HOST: https://api.dev.zoo.dev
@@ -178,19 +178,17 @@ jobs:
         shell: bash
         run: |-
           cp nextest-archive.tar.zst rust/nextest-archive.tar.zst
-          cd rust
+          pushd rust
           cargo nextest run \
-           --retries=10 --no-fail-fast --profile=ci --archive-file nextest-archive.tar.zst \
-           --partition count:${{ matrix.partitionIndex}}/${{ matrix.partitionTotal }} \
-           2>&1 | tee /tmp/github-actions.log
+            --retries=10 --no-fail-fast --profile=ci --archive-file nextest-archive.tar.zst \
+            --partition count:${{ matrix.partitionIndex}}/${{ matrix.partitionTotal }} \
+            2>&1 | tee /tmp/github-actions.log || true  # let TAB determine failure
+          popd
+          .github/ci-cd-scripts/upload-results.sh
         env:
           KITTYCAD_API_TOKEN: ${{secrets.KITTYCAD_API_TOKEN_DEV}}
           ZOO_HOST: https://api.dev.zoo.dev
           RUST_MIN_STACK: 10485760000
-      - name: Upload results
-        if: always()
-        run: .github/ci-cd-scripts/upload-results.sh
-        env:
           TAB_API_URL: ${{ secrets.TAB_API_URL }}
           TAB_API_KEY: ${{ secrets.TAB_API_KEY }}
           CI_COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
@@ -231,9 +229,9 @@ jobs:
         run: |-
           cd rust/kcl-lib
           cargo nextest run \
-           --retries=10 --no-fail-fast --features=artifact-graph --profile=ci \
-           internal \
-           2>&1 | tee /tmp/github-actions.log
+            --retries=10 --no-fail-fast --features=artifact-graph --profile=ci \
+            internal \
+            2>&1 | tee /tmp/github-actions.log
         env:
           TWENTY_TWENTY: overwrite
           INSTA_UPDATE: always