Let's see if we can collate skipped test info and fail the workflow run if all test configurations cause any tests to be skipped.

Author: johnbillion

.github/scripts/analyze-skipped-tests.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Analyze skipped tests across all PHPUnit configurations
+# Fails if any tests are skipped in ALL configurations (never run)
+
+# Count total test runs
+TOTAL_TEST_RUNS=$(ls -1 skipped-tests-*.txt 2>/dev/null | wc -l)
+
+if [ "$TOTAL_TEST_RUNS" -eq 0 ]; then
+  echo "ERROR: No skipped test artifacts found"
+  exit 1
+fi
+
+# Collect all skipped tests from all runs
+grep -E '^[A-Za-z0-9_\\]+::[A-Za-z0-9_]+$' skipped-tests-*.txt 2>/dev/null | cut -d: -f2- | sort | uniq > unique-skipped-tests.txt
+
+# Find tests that are skipped in ALL runs
+ALWAYS_SKIPPED_TESTS=""
+while IFS= read -r test; do
+  if [ -n "$test" ]; then
+    # Count how many files contain this test
+    count=$(grep -l "^${test}$" skipped-tests-*.txt | wc -l)
+
+    # If skipped in all runs, add to list
+    if [ "$count" -eq "$TOTAL_TEST_RUNS" ]; then
+      ALWAYS_SKIPPED_TESTS="${ALWAYS_SKIPPED_TESTS}${test}\n"
+    fi
+  fi
+done < unique-skipped-tests.txt
+
+if [ -n "$ALWAYS_SKIPPED_TESTS" ]; then
+  echo "ERROR: Found tests that are never run in any configuration:"
+  echo -e "$ALWAYS_SKIPPED_TESTS"
+  exit 1
+fi

.github/scripts/extract-skipped-tests.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Extract skipped tests from PHPUnit JUnit XML output
+# Usage: extract-skipped-tests.sh <php> <db-type> <db-version> <multisite> <memcached> <test-groups>
+
+# Create config ID from parameters
+CONFIG_ID="php${1}_${2}${3}_${4}_${5}_${6}"
+CONFIG_ID="${CONFIG_ID//[^a-zA-Z0-9_-]/_}"
+
+echo "Extracting skipped tests for configuration: ${CONFIG_ID}"
+
+# Extract skipped tests from XML files using xmllint
+touch "skipped-tests-${CONFIG_ID}.txt"
+
+for file in phpunit-results-*.xml; do
+  if [ -f "$file" ]; then
+    # Extract skipped tests in class::method format
+    xmllint --format "$file" 2>/dev/null | grep -B1 "<skipped" | grep "testcase" | sed -n 's/.*name="\([^"]*\)".*classname="\([^"]*\)".*/\2::\1/p' >> "skipped-tests-${CONFIG_ID}.txt" || true
+  fi
+done
+
+echo "Skipped tests saved to skipped-tests-${CONFIG_ID}.txt"
+echo "CONFIG_ID=${CONFIG_ID}"

.github/workflows/phpunit-tests.yml

@@ -311,6 +311,29 @@ jobs:
       memcached: ${{ matrix.memcached || false }}
       phpunit-test-groups: ${{ matrix.phpunit-test-groups || '' }}
 
+  analyze-skipped-tests:
+    name: Analyze Skipped Tests
+    runs-on: ubuntu-24.04
+    needs:
+      - test-with-mysql
+      - test-with-mariadb
+      - test-innovation-releases
+      - html-api-test-groups
+    permissions:
+      contents: read
+    steps:
+      - name: Download skipped test artifacts
+        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
+        with:
+          pattern: skipped-tests-*
+          path: skipped-tests-artifacts
+          merge-multiple: true
+
+      - name: Analyze skipped tests
+        run: |
+          cd skipped-tests-artifacts
+          ../.github/scripts/analyze-skipped-tests.sh
+
   slack-notifications:
     name: Slack Notifications
     uses: ./.github/workflows/slack-notifications.yml

.github/workflows/reusable-phpunit-tests-v3.yml

@@ -211,6 +211,7 @@ jobs:
             php ./vendor/bin/phpunit \
             --verbose \
             -c "${PHPUNIT_CONFIG}" \
+            --log-junit "phpunit-results-${MULTISITE_FLAG}.xml" \
             ${{ inputs.phpunit-test-groups && '--group "${TEST_GROUPS}"' || '' }} \
             ${{ inputs.coverage-report && '--coverage-clover "wp-code-coverage-${MULTISITE_FLAG}-${GITHUB_SHA}.xml" --coverage-html "wp-code-coverage-${MULTISITE_FLAG}-${GITHUB_SHA}"' || '' }}
         env:
@@ -220,23 +221,58 @@ jobs:
       - name: Run AJAX tests
         if: ${{ ! inputs.phpunit-test-groups && ! inputs.coverage-report }}
         continue-on-error: ${{ inputs.allow-errors }}
-        run: node ./tools/local-env/scripts/docker.js run php ./vendor/bin/phpunit --verbose -c "${PHPUNIT_CONFIG}" --group ajax
+        run: node ./tools/local-env/scripts/docker.js run php ./vendor/bin/phpunit --verbose -c "${PHPUNIT_CONFIG}" --group ajax --log-junit "phpunit-results-ajax-${MULTISITE_FLAG}.xml"
+        env:
+          MULTISITE_FLAG: ${{ inputs.multisite && 'multisite' || 'single' }}
 
       - name: Run ms-files tests as a multisite install
         if: ${{ inputs.multisite && ! inputs.phpunit-test-groups && ! inputs.coverage-report }}
         continue-on-error: ${{ inputs.allow-errors }}
-        run: node ./tools/local-env/scripts/docker.js run php ./vendor/bin/phpunit --verbose -c "${PHPUNIT_CONFIG}" --group ms-files
+        run: node ./tools/local-env/scripts/docker.js run php ./vendor/bin/phpunit --verbose -c "${PHPUNIT_CONFIG}" --group ms-files --log-junit "phpunit-results-ms-files.xml"
 
       - name: Run external HTTP tests
         if: ${{ ! inputs.multisite && ! inputs.phpunit-test-groups && ! inputs.coverage-report }}
         continue-on-error: ${{ inputs.allow-errors }}
-        run: node ./tools/local-env/scripts/docker.js run php ./vendor/bin/phpunit --verbose -c "${PHPUNIT_CONFIG}" --group external-http
+        run: node ./tools/local-env/scripts/docker.js run php ./vendor/bin/phpunit --verbose -c "${PHPUNIT_CONFIG}" --group external-http --log-junit "phpunit-results-external-http.xml"
 
       # __fakegroup__ is excluded to force PHPUnit to ignore the <exclude> settings in phpunit.xml.dist.
       - name: Run (Xdebug) tests
         if: ${{ inputs.php != '8.4' && ! inputs.phpunit-test-groups && ! inputs.coverage-report }}
         continue-on-error: ${{ inputs.allow-errors }}
-        run: LOCAL_PHP_XDEBUG=true node ./tools/local-env/scripts/docker.js run php ./vendor/bin/phpunit -v --group xdebug --exclude-group __fakegroup__
+        run: LOCAL_PHP_XDEBUG=true node ./tools/local-env/scripts/docker.js run php ./vendor/bin/phpunit -v --group xdebug --exclude-group __fakegroup__ --log-junit "phpunit-results-xdebug.xml"
+
+      - name: Extract skipped tests from JUnit XML
+        id: extract-skipped
+        env:
+          PHP_VERSION: ${{ inputs.php }}
+          DB_TYPE: ${{ inputs.db-type }}
+          DB_VERSION: ${{ inputs.db-version }}
+          MULTISITE: ${{ inputs.multisite && 'multisite' || 'single' }}
+          MEMCACHED: ${{ inputs.memcached && 'memcached' || 'no-memcached' }}
+          TEST_GROUPS: ${{ inputs.phpunit-test-groups && inputs.phpunit-test-groups || 'all-groups' }}
+        run: |
+          CONFIG_OUTPUT=$(./.github/scripts/extract-skipped-tests.sh \
+            "${PHP_VERSION}" \
+            "${DB_TYPE}" \
+            "${DB_VERSION}" \
+            "${MULTISITE}" \
+            "${MEMCACHED}" \
+            "${TEST_GROUPS}")
+
+          # Extract CONFIG_ID from the script output
+          CONFIG_ID=$(echo "$CONFIG_OUTPUT" | grep "CONFIG_ID=" | cut -d= -f2)
+          echo "CONFIG_ID=${CONFIG_ID}" >> $GITHUB_OUTPUT
+
+      - name: Upload skipped tests summary
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: skipped-tests-${{ steps.extract-skipped.outputs.CONFIG_ID }}
+          path: skipped-tests-*.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+      - name: Clean up test artifacts
+        run: rm -f phpunit-results-*.xml skipped-tests-*.txt
 
       - name: Upload test coverage report to Codecov
         if: ${{ inputs.coverage-report }}