End-to-end tests #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: End-to-end tests | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| - release-*.*.* | |
| # TODO: re-enable this before merging | |
| # concurrency: | |
| # group: ${{ github.workflow }}-${{ github.ref }} | |
| # cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} | |
| permissions: {} | |
| env: | |
| ACTIONS_STEP_DEBUG: true | |
| RUNNER_DEBUG: 1 | |
| jobs: | |
| detect-changes: | |
| name: Detect whether code changed | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| outputs: | |
| changed: ${{ steps.detect-changes.outputs.e2e }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: true # required to get more history in the changed-files action | |
| fetch-depth: 2 | |
| - name: Detect changes | |
| id: detect-changes | |
| uses: ./.github/actions/change-detection | |
| with: | |
| self: .github/workflows/pr-e2e-tests.yml | |
| build-grafana: | |
| needs: detect-changes | |
| if: needs.detect-changes.outputs.changed == 'true' | |
| name: Build & Package Grafana | |
| runs-on: ubuntu-latest-16-cores | |
| permissions: | |
| contents: read | |
| outputs: | |
| artifact: ${{ steps.artifact.outputs.artifact }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| # TODO: add a cleanup workflow to remove the cache when the PR is closed | |
| # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#force-deletion-of-caches-overriding-default-cache-eviction-policy | |
| # TODO: maybe we could just use the cache to store the build, instead of uploading as an artifact? | |
| - uses: actions/cache@v4 | |
| id: cache | |
| with: | |
| key: "build-grafana-${{ runner.os }}-${{ hashFiles('yarn.lock', 'public/*', 'packages/*', 'pkg/**/*.go', '**/go.mod', '**/go.sum', '!**_test.go', '!**.test.ts', '!**.test.tsx') }}" | |
| path: | | |
| build-dir | |
| # If no cache hit, build Grafana | |
| - name: Build Grafana | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e | |
| with: | |
| verb: run | |
| args: go run ./pkg/build/cmd artifacts -a targz:grafana:linux/amd64 --grafana-dir="${PWD}" > out.txt | |
| - name: Cat built artifact | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| run: cat out.txt | |
| - name: Move built artifact | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| run: | | |
| mkdir -p build-dir | |
| mv "$(cat out.txt)" build-dir/grafana.tar.gz | |
| # If cache hit, validate the artifact is present | |
| - name: Validate artifact | |
| if: steps.cache.outputs.cache-hit == 'true' | |
| run: | | |
| if [ ! -f build-dir/grafana.tar.gz ]; then | |
| echo "Error: build-dir/grafana.tar.gz not found in cache" | |
| exit 1 | |
| fi | |
| - name: Set artifact name | |
| run: echo "artifact=grafana-server-${{github.run_number}}" >> "$GITHUB_OUTPUT" | |
| id: artifact | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| id: upload | |
| with: | |
| retention-days: 1 | |
| name: ${{ steps.artifact.outputs.artifact }} | |
| path: build-dir/grafana.tar.gz | |
| # TODO: we won't need this when we only have playwright | |
| build-e2e-runner: | |
| needs: detect-changes | |
| if: needs.detect-changes.outputs.changed == 'true' | |
| name: Build E2E test runner | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| outputs: | |
| artifact: ${{ steps.artifact.outputs.artifact }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Setup Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: go.mod | |
| cache: ${{ !github.event.pull_request.head.repo.fork }} | |
| - name: Build E2E test runner | |
| id: artifact | |
| run: | | |
| set -euo pipefail | |
| # We want a static binary, so we need to set CGO_ENABLED=0 | |
| CGO_ENABLED=0 go build -o ./e2e-runner ./e2e/ | |
| echo "artifact=e2e-runner-${{github.run_number}}" >> "$GITHUB_OUTPUT" | |
| - uses: actions/upload-artifact@v4 | |
| id: upload | |
| with: | |
| retention-days: 1 | |
| name: ${{ steps.artifact.outputs.artifact }} | |
| path: e2e-runner | |
| run-e2e-tests: | |
| needs: | |
| - build-grafana | |
| - build-e2e-runner | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - suite: various-suite | |
| path: e2e/various-suite | |
| - suite: dashboards-suite | |
| path: e2e/dashboards-suite | |
| - suite: smoke-tests-suite | |
| path: e2e/smoke-tests-suite | |
| - suite: panels-suite | |
| path: e2e/panels-suite | |
| - suite: various-suite (old arch) | |
| path: e2e/old-arch/various-suite | |
| flags: --flags="--env dashboardScene=false" | |
| - suite: dashboards-suite (old arch) | |
| path: e2e/old-arch/dashboards-suite | |
| flags: --flags="--env dashboardScene=false" | |
| - suite: smoke-tests-suite (old arch) | |
| path: e2e/old-arch/smoke-tests-suite | |
| flags: --flags="--env dashboardScene=false" | |
| - suite: panels-suite (old arch) | |
| path: e2e/old-arch/panels-suite | |
| flags: --flags="--env dashboardScene=false" | |
| name: ${{ matrix.suite }} | |
| runs-on: ubuntu-latest-8-cores | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ needs.build-grafana.outputs.artifact }} | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ needs.build-e2e-runner.outputs.artifact }} | |
| - name: chmod +x | |
| run: chmod +x ./e2e-runner | |
| - name: Run E2E tests | |
| uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e | |
| with: | |
| verb: run | |
| args: go run ./pkg/build/e2e --package=grafana.tar.gz | |
| --suite=${{ matrix.path }} | |
| ${{ matrix.flags }} | |
| - name: Set suite name | |
| id: set-suite-name | |
| if: success() || failure() | |
| env: | |
| SUITE: ${{ matrix.path }} | |
| run: | | |
| set -euo pipefail | |
| echo "suite=$(echo "$SUITE" | sed 's/\//-/g')" >> "$GITHUB_OUTPUT" | |
| - uses: actions/upload-artifact@v4 | |
| if: success() || failure() | |
| with: | |
| name: ${{ steps.set-suite-name.outputs.suite }}-${{ github.run_number }} | |
| path: videos | |
| retention-days: 1 | |
| run-playwright-tests: | |
| needs: | |
| - build-grafana | |
| name: Playwright E2E tests (${{ matrix.shard }}/${{ matrix.shardTotal }}) | |
| runs-on: ubuntu-latest-8-cores | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: [1, 2, 3, 4, 5, 6, 7, 8] | |
| shardTotal: [8] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ needs.build-grafana.outputs.artifact }} | |
| - name: Run E2E tests | |
| uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e | |
| with: | |
| verb: run | |
| args: go run ./pkg/build/e2e-playwright --package=grafana.tar.gz --shard=${{ matrix.shard }}/${{ matrix.shardTotal }} --blob-dir=./blob-report | |
| - uses: actions/upload-artifact@v4 | |
| if: success() || failure() | |
| with: | |
| name: playwright-blob-${{ github.run_number }}-${{ matrix.shard }} | |
| path: ./blob-report | |
| retention-days: 1 | |
| required-playwright-tests: | |
| needs: | |
| - run-playwright-tests | |
| - build-grafana | |
| if: ${{ !cancelled() }} | |
| name: All Playwright tests complete | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version-file: '.nvmrc' | |
| - name: Download blob reports from GitHub Actions Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: blobs | |
| pattern: playwright-blob-* | |
| merge-multiple: true | |
| - name: Check blob reports | |
| run: | | |
| if [ ! "$(ls -A ./blobs)" ]; then | |
| echo "Error: No blob reports found in ./blobs directory" | |
| echo "Did the Playwright tests run at all?" | |
| exit 1 | |
| fi | |
| echo "Found blob reports in ./blobs:" | |
| ls -lah ./blobs | |
| - name: Merge into HTML Report | |
| run: npx playwright merge-reports --reporter html ./blobs | |
| - name: Merge into JSON Report | |
| env: | |
| PLAYWRIGHT_JSON_OUTPUT_NAME: /tmp/playwright-results.json | |
| run: npx playwright merge-reports --reporter=json ./blobs | |
| - name: Bench report | |
| run: | | |
| docker run --rm \ | |
| --volume="/tmp/playwright-results.json:/home/bench/tests/playwright-results.json" \ | |
| us-docker.pkg.dev/grafanalabs-global/docker-grafana-bench-prod/grafana-bench:v0.5.1 report \ | |
| --grafana-url "http://localhost:3000" \ | |
| --grafana-version "CI- ${{ github.sha }}" \ | |
| --test-suite-name "FrontendCore" \ | |
| --report-input playwright \ | |
| --report-output log \ | |
| --log-level DEBUG \ | |
| /home/bench/tests/playwright-results.json | |
| - name: Upload HTML report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: playwright-html-${{ github.run_number }} | |
| path: playwright-report | |
| retention-days: 7 | |
| - name: Check test suites | |
| id: check-jobs | |
| uses: ./.github/actions/check-jobs | |
| continue-on-error: true # Failure will be reported on Show test results step | |
| with: | |
| needs: ${{ toJson(needs) }} | |
| failure-message: "One or more E2E test suites have failed" | |
| success-message: "All E2E test suites completed successfully" | |
| - name: Show test results | |
| env: | |
| FAILED: ${{ steps.check-jobs.outputs.any-failed }} | |
| # sed removes the leading `../../src/` from the paths | |
| run: | | |
| npx playwright merge-reports --reporter list ./all-blob-reports | sed 's|\(\.\./\)\{1,\}src/|/|g' | |
| if [ "$FAILED" = "true" ]; then | |
| exit 1 | |
| fi | |
| run-a11y-test: | |
| needs: | |
| - build-grafana | |
| name: A11y test | |
| runs-on: ubuntu-latest-8-cores | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ needs.build-grafana.outputs.artifact }} | |
| - name: Run PR a11y test | |
| if: github.event_name == 'pull_request' | |
| uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e | |
| with: | |
| verb: run | |
| args: go run ./pkg/build/a11y --package=grafana.tar.gz | |
| - name: Run non-PR a11y test | |
| if: github.event_name != 'pull_request' | |
| uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e | |
| with: | |
| verb: run | |
| args: go run ./pkg/build/a11y --package=grafana.tar.gz --no-threshold-fail | |
| # This is the job that is actually required by rulesets. | |
| # We want to only require one job instead of all the individual tests. | |
| # Future work also allows us to start skipping some tests based on changed files. | |
| required-e2e-tests: | |
| needs: | |
| - run-e2e-tests | |
| - build-grafana | |
| # a11y test is not listed on purpose: it is not an important E2E test. | |
| # It is also totally fine to fail right now. | |
| # always() is the best function here. | |
| # success() || failure() will skip this function if any need is also skipped. | |
| # That means conditional test suites will fail the entire requirement check. | |
| if: always() | |
| name: All E2E tests complete | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Check test suites | |
| uses: ./.github/actions/check-jobs | |
| with: | |
| needs: ${{ toJson(needs) }} | |
| failure-message: "One or more E2E test suites have failed" | |
| success-message: "All E2E test suites completed successfully" |