Accessibility Regression Gate #242
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Accessibility Regression Gate | |
| on: | |
| pull_request: | |
| paths: | |
| - "web/**" | |
| - ".github/workflows/accessibility-regression.yml" | |
| - "scripts/pre-commit-check.py" | |
| - ".githooks/pre-commit" | |
| push: | |
| branches: [main] | |
| paths: | |
| - "web/**" | |
| - ".github/workflows/accessibility-regression.yml" | |
| - "scripts/pre-commit-check.py" | |
| - ".githooks/pre-commit" | |
| schedule: | |
| - cron: "0 9 * * 1" | |
| permissions: | |
| contents: read | |
| security-events: write | |
| jobs: | |
| accessibility-regression: | |
| name: Playwright + axe gate | |
| runs-on: ubuntu-latest | |
| env: | |
| E2E_PORT: "5100" | |
| E2E_BASE_URL: "http://127.0.0.1:5100" | |
| E2E_AXE_FAIL_INCOMPLETE: "1" | |
| GLOW_BYPASS_CONSENT_FOR_AUTOMATION: "1" | |
| GLOW_ENABLE_AUTOMATION_CONSENT_ENDPOINT: "1" | |
| GLOW_AUTOMATION_CONSENT_TOKEN: "GLOW" | |
| GLOW_ENABLE_AI: "false" | |
| GLOW_ENABLE_AI_GENERAL_CHAT: "false" | |
| GLOW_ENABLE_AI_CHAT: "false" | |
| GLOW_ENABLE_AI_WHISPERER: "false" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v5 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.13" | |
| - name: Set up Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "22" | |
| - name: Install system deps | |
| run: sudo apt-get install -y pandoc | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install --upgrade --force-reinstall "git+https://github.com/Community-Access/quill-glow-core.git@main" | |
| python -m pip install -e ./desktop | |
| python -m pip install -e ./web | |
| - name: Install web Node dependencies | |
| working-directory: web | |
| run: npm ci | |
| - name: Install Playwright browser | |
| working-directory: web | |
| run: npx playwright install --with-deps chromium | |
| - name: Run Site Audit fast-lane regression | |
| working-directory: web | |
| run: npm run test:e2e:site-audit | |
| - name: Run Playwright regression suite | |
| working-directory: web | |
| run: npm run test:e2e | |
| - name: Run Playwright axe-core audit (all 17 routes + interactive states) | |
| working-directory: web | |
| run: npm run test:axe | |
| env: | |
| E2E_PORT: "5100" | |
| E2E_BASE_URL: "http://127.0.0.1:5100" | |
| E2E_AXE_FAIL_INCOMPLETE: "1" | |
| - name: Run weekly strict full axe-core audit | |
| if: github.event_name == 'schedule' | |
| working-directory: web | |
| run: npm run test:axe:strict | |
| env: | |
| E2E_PORT: "5100" | |
| E2E_BASE_URL: "http://127.0.0.1:5100" | |
| E2E_AXE_STRICT: "1" | |
| E2E_AXE_FAIL_INCOMPLETE: "1" | |
| - name: Run axe CLI scan and emit SARIF | |
| if: always() | |
| shell: bash | |
| run: | | |
| set -uo pipefail | |
| mkdir -p web/e2e/artifacts | |
| GLOW_BYPASS_CONSENT_FOR_AUTOMATION=1 GLOW_ENABLE_AUTOMATION_CONSENT_ENDPOINT=1 GLOW_AUTOMATION_CONSENT_TOKEN=GLOW python -m flask --app acb_large_print_web.app:create_app run --no-debugger --no-reload --port 5100 > web/e2e/artifacts/flask.log 2>&1 & | |
| FLASK_PID=$! | |
| ready=0 | |
| for i in {1..30}; do | |
| if curl -fsS http://127.0.0.1:5100/health > /dev/null; then | |
| ready=1 | |
| break | |
| fi | |
| sleep 2 | |
| done | |
| if [ "$ready" != "1" ]; then | |
| echo "::error::Flask did not become ready on port 5100 within 60s" | |
| cat web/e2e/artifacts/flask.log || true | |
| kill "$FLASK_PID" 2>/dev/null || true | |
| # Emit an empty axe JSON so the SARIF converter still runs and the | |
| # SARIF upload step has a file to consume. | |
| echo "[]" > web/e2e/artifacts/axe-results.json | |
| else | |
| # @axe-core/cli exits non-zero when it finds violations. That is the | |
| # *expected* outcome of a regression gate: we still want a JSON file | |
| # so we can convert it to SARIF and upload it to code scanning. | |
| # The actual gating happens after SARIF upload (see "Gate" step). | |
| npx --no-install axe \ | |
| http://127.0.0.1:5100/ \ | |
| http://127.0.0.1:5100/audit/ \ | |
| http://127.0.0.1:5100/fix/ \ | |
| http://127.0.0.1:5100/convert/ \ | |
| http://127.0.0.1:5100/template/ \ | |
| http://127.0.0.1:5100/speech/ \ | |
| http://127.0.0.1:5100/braille/ \ | |
| http://127.0.0.1:5100/settings/ \ | |
| http://127.0.0.1:5100/guidelines/ \ | |
| http://127.0.0.1:5100/guide/ \ | |
| http://127.0.0.1:5100/about/ \ | |
| http://127.0.0.1:5100/changelog/ \ | |
| http://127.0.0.1:5100/faq/ \ | |
| http://127.0.0.1:5100/rules/ \ | |
| http://127.0.0.1:5100/feedback/ \ | |
| http://127.0.0.1:5100/privacy/ \ | |
| http://127.0.0.1:5100/status/ \ | |
| --tags wcag2a,wcag2aa,wcag21a,wcag21aa,wcag22aa \ | |
| --save web/e2e/artifacts/axe-results.json || echo "axe reported issues (continuing to SARIF conversion)" | |
| fi | |
| # Always produce a SARIF, even if the JSON is empty or malformed, | |
| # so the upload step succeeds and we can inspect results. | |
| if [ ! -s web/e2e/artifacts/axe-results.json ]; then | |
| echo "[]" > web/e2e/artifacts/axe-results.json | |
| fi | |
| if ! python .github/scripts/axe_json_to_sarif.py \ | |
| web/e2e/artifacts/axe-results.json \ | |
| web/e2e/artifacts/axe-results.sarif; then | |
| echo "::warning::SARIF conversion failed; emitting empty SARIF" | |
| python -c "import json; json.dump({'version':'2.1.0','\$schema':'https://json.schemastore.org/sarif-2.1.0.json','runs':[{'tool':{'driver':{'name':'axe-core','informationUri':'https://github.com/dequelabs/axe-core-npm','rules':[]}},'results':[]}]}, open('web/e2e/artifacts/axe-results.sarif','w'))" | |
| fi | |
| kill "$FLASK_PID" 2>/dev/null || true | |
| - name: Upload SARIF | |
| if: always() | |
| uses: github/codeql-action/upload-sarif@v3 | |
| with: | |
| sarif_file: web/e2e/artifacts/axe-results.sarif | |
| category: accessibility-regression | |
| - name: Upload accessibility artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: accessibility-regression-artifacts | |
| path: web/e2e/artifacts/ | |
| retention-days: 14 | |
| - name: Gate on critical / serious accessibility violations | |
| if: always() | |
| shell: bash | |
| run: | | |
| set -uo pipefail | |
| python - <<'PY' | |
| import json, sys, pathlib | |
| p = pathlib.Path("web/e2e/artifacts/axe-results.json") | |
| if not p.exists(): | |
| print("No axe-results.json present; treating as failure.") | |
| sys.exit(1) | |
| try: | |
| data = json.loads(p.read_text(encoding="utf-8")) | |
| except Exception as e: | |
| print(f"Could not parse axe-results.json: {e}") | |
| sys.exit(1) | |
| pages = data if isinstance(data, list) else [data] | |
| blocking = [] | |
| for page in pages: | |
| for v in page.get("violations", []): | |
| impact = (v.get("impact") or "").lower() | |
| if impact in {"critical", "serious"}: | |
| blocking.append((page.get("url", "?"), v.get("id"), impact, len(v.get("nodes", [])))) | |
| if blocking: | |
| print("Critical / serious accessibility violations:") | |
| for url, rule, impact, n in blocking: | |
| print(f" - [{impact}] {rule} ({n} nodes) on {url}") | |
| sys.exit(1) | |
| print("No critical / serious accessibility violations.") | |
| PY |