Production regression tests #990
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Production regression tests | |
| on: | |
| schedule: | |
| - cron: "45 20 * * 1-6" | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| issues: read | |
| packages: none | |
| env: | |
| SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} | |
| SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} | |
| SAUCE_CAPABILITIES_OVERRIDES_PATH: "sauceLabsCapabilities.json" | |
| SAUCE_JOB: "Alloy Prod Workflow" | |
| ALLOY_ENV: prod | |
| jobs: | |
| get-testing-tags: | |
| name: Fetch releases | |
| runs-on: ubuntu-latest | |
| outputs: | |
| MATRIX_INPUT: ${{ steps.retrieve-tags.outputs.MATRIX_INPUT }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: pnpm/action-setup@v4 | |
| with: | |
| version: 10 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v5 | |
| with: | |
| node-version-file: .nvmrc | |
| - run: pnpm i --frozen-lockfile | |
| - name: Upload sauce configs | |
| uses: actions/upload-artifact@v4.4.0 | |
| with: | |
| name: sauce-config | |
| path: .sauce | |
| include-hidden-files: true | |
| - name: Upload sauce ignore | |
| uses: actions/upload-artifact@v4.4.0 | |
| with: | |
| name: sauce-ignore | |
| path: .sauceignore | |
| include-hidden-files: true | |
| - name: Retrieve tags | |
| id: retrieve-tags | |
| run: |- | |
| TAGS_TO_TEST=$(node -e "import('./scripts/getTestingTags.js').then(m => m.default().then(tags => console.log(JSON.stringify(tags))))") | |
| echo "Tags to test: $TAGS_TO_TEST" | |
| MATRIX_INPUT="{\"include\":$TAGS_TO_TEST}" | |
| echo "MATRIX_INPUT=$MATRIX_INPUT" >> "$GITHUB_OUTPUT" | |
| alloy-prod-e2e-safari: | |
| name: Safari | |
| needs: get-testing-tags | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| strategy: | |
| max-parallel: 10 | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.get-testing-tags.outputs.MATRIX_INPUT) }} | |
| continue-on-error: true | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ matrix.tag.tag }} | |
| - uses: pnpm/action-setup@v4 | |
| with: | |
| version: 10 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v5 | |
| with: | |
| node-version-file: .nvmrc | |
| - name: Install dependencies | |
| run: pnpm install --frozen-lockfile | |
| - name: Store Alloy version | |
| uses: nyaa8/package-version@v1 | |
| - name: Download sauce configs | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: sauce-config | |
| path: .sauce | |
| - name: Download sauce ignore | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: sauce-ignore | |
| path: . | |
| - name: Build | |
| run: | | |
| echo "PACKAGE_VERSION=$(pnpm pkg get version | tr -d '\"')" >> $GITHUB_ENV | |
| pnpm run test:functional:build:prod | |
| env: | |
| ALLOY_PROD_VERSION: ${{ env.PACKAGE_VERSION }} | |
| BASE_CODE_MIN: true | |
| NPM_PACKAGE_PROD: true | |
| - name: Update Sauce Config with Node | |
| run: | | |
| sed -i 's/"$NODE_VERSION"/"${{ matrix.tag.nodeVersion }}"/g' .sauce/prod-safari.yml | |
| - name: Run Functional Tests | |
| id: saucectl | |
| uses: saucelabs/saucectl-run-action@v4.3.0 | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| with: | |
| sauce-username: ${{ secrets.SAUCE_USERNAME }} | |
| sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} | |
| env: | | |
| SAUCE_USERNAME=${{ secrets.SAUCE_USERNAME }} | |
| SAUCE_ACCESS_KEY=${{ secrets.SAUCE_ACCESS_KEY }} | |
| ALLOY_PROD_VERSION=${{ env.PACKAGE_VERSION }} | |
| NPM_PACKAGE_VERSION=${{ env.PACKAGE_VERSION }} | |
| ALLOY_ENV=prod | |
| config-file: ./.sauce/prod-safari.yml | |
| - name: Get Saucelabs Test Results | |
| if: failure() | |
| id: saucelabs-results | |
| env: | |
| SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} | |
| SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} | |
| run: | | |
| mkdir -p test-results | |
| if [ -d "./test-results" ] && [ "$(ls -A ./test-results)" ]; then | |
| cp -r ./test-results/* ./test-results/ | |
| echo "TEST_RESULTS<<EOF" >> $GITHUB_ENV | |
| cat ./test-results/* >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| else | |
| echo "TEST_RESULTS=No test results found in ./test-results/" >> $GITHUB_ENV | |
| fi | |
| - name: Handle Test Failures | |
| if: failure() | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{secrets.GITHUB_TOKEN}} | |
| script: | | |
| const handleTestFailures = require('./.github/scripts/handle-test-failures.js'); | |
| await handleTestFailures({ github, context, browser: 'safari' }); | |
| alloy-prod-e2e-firefox: | |
| name: Firefox | |
| needs: get-testing-tags | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| strategy: | |
| max-parallel: 10 | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.get-testing-tags.outputs.MATRIX_INPUT) }} | |
| continue-on-error: true | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ matrix.tag.tag }} | |
| - uses: pnpm/action-setup@v4 | |
| with: | |
| version: 10 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v5 | |
| with: | |
| node-version-file: .nvmrc | |
| - name: Install dependencies | |
| run: pnpm install --frozen-lockfile | |
| - name: Store Alloy version | |
| uses: nyaa8/package-version@v1 | |
| - name: Download sauce configs | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: sauce-config | |
| path: .sauce | |
| - name: Download sauce ignore | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: sauce-ignore | |
| path: . | |
| - name: Build | |
| run: | | |
| echo "PACKAGE_VERSION=$(pnpm pkg get version | tr -d '\"')" >> $GITHUB_ENV | |
| pnpm run test:functional:build:prod | |
| env: | |
| ALLOY_PROD_VERSION: ${{ env.PACKAGE_VERSION }} | |
| BASE_CODE_MIN: true | |
| NPM_PACKAGE_PROD: true | |
| - name: Update Sauce Config with Node | |
| run: | | |
| sed -i 's/"$NODE_VERSION"/"${{ matrix.tag.nodeVersion }}"/g' .sauce/prod-firefox.yml | |
| - name: Run Functional Tests | |
| id: saucectl | |
| uses: saucelabs/saucectl-run-action@v4.3.0 | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| with: | |
| sauce-username: ${{ secrets.SAUCE_USERNAME }} | |
| sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} | |
| env: | | |
| SAUCE_USERNAME=${{ secrets.SAUCE_USERNAME }} | |
| SAUCE_ACCESS_KEY=${{ secrets.SAUCE_ACCESS_KEY }} | |
| ALLOY_PROD_VERSION=${{ env.PACKAGE_VERSION }} | |
| NPM_PACKAGE_VERSION=${{ env.PACKAGE_VERSION }} | |
| ALLOY_ENV=prod | |
| config-file: ./.sauce/prod-firefox.yml | |
| - name: Get Saucelabs Test Results | |
| if: failure() | |
| id: saucelabs-results | |
| env: | |
| SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} | |
| SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} | |
| run: | | |
| mkdir -p test-results | |
| if [ -d "./test-results" ] && [ "$(ls -A ./test-results)" ]; then | |
| cp -r ./test-results/* ./test-results/ | |
| echo "TEST_RESULTS<<EOF" >> $GITHUB_ENV | |
| cat ./test-results/* >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| else | |
| echo "TEST_RESULTS=No test results found in ./test-results/" >> $GITHUB_ENV | |
| fi | |
| - name: Handle Test Failures | |
| if: failure() | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{secrets.GITHUB_TOKEN}} | |
| script: | | |
| const handleTestFailures = require('./.github/scripts/handle-test-failures.js'); | |
| await handleTestFailures({ github, context, browser: 'firefox' }); | |
| alloy-prod-e2e-chrome: | |
| name: Chrome | |
| needs: get-testing-tags | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| strategy: | |
| max-parallel: 10 | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.get-testing-tags.outputs.MATRIX_INPUT) }} | |
| continue-on-error: true | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ matrix.tag.tag }} | |
| - uses: pnpm/action-setup@v4 | |
| with: | |
| version: 10 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v5 | |
| with: | |
| node-version-file: .nvmrc | |
| - name: Install dependencies | |
| run: pnpm install --frozen-lockfile | |
| - name: Store Alloy version | |
| uses: nyaa8/package-version@v1 | |
| - name: Download sauce configs | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: sauce-config | |
| path: .sauce | |
| - name: Download sauce ignore | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: sauce-ignore | |
| path: . | |
| - name: Build | |
| run: | | |
| echo "PACKAGE_VERSION=$(pnpm pkg get version | tr -d '\"')" >> $GITHUB_ENV | |
| pnpm run test:functional:build:prod | |
| env: | |
| ALLOY_PROD_VERSION: ${{ env.PACKAGE_VERSION }} | |
| BASE_CODE_MIN: true | |
| NPM_PACKAGE_PROD: true | |
| - name: Update Sauce Config with Node | |
| run: | | |
| sed -i 's/"$NODE_VERSION"/"${{ matrix.tag.nodeVersion }}"/g' .sauce/prod-chrome.yml | |
| - name: Run Functional Tests | |
| id: saucectl | |
| uses: saucelabs/saucectl-run-action@v4.3.0 | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| with: | |
| sauce-username: ${{ secrets.SAUCE_USERNAME }} | |
| sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} | |
| env: | | |
| SAUCE_USERNAME=${{ secrets.SAUCE_USERNAME }} | |
| SAUCE_ACCESS_KEY=${{ secrets.SAUCE_ACCESS_KEY }} | |
| ALLOY_PROD_VERSION=${{ env.PACKAGE_VERSION }} | |
| NPM_PACKAGE_VERSION=${{ env.PACKAGE_VERSION }} | |
| ALLOY_ENV=prod | |
| config-file: ./.sauce/prod-chrome.yml | |
| - name: Handle Test Failures | |
| if: failure() | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{secrets.GITHUB_TOKEN}} | |
| script: | | |
| const handleTestFailures = require('./.github/scripts/handle-test-failures.js'); | |
| await handleTestFailures({ github, context, browser: 'chrome' }); |