This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: TestDriver.ai / Generate / Regressions | |
| permissions: | |
| actions: write | |
| contents: write | |
| statuses: write | |
| pull-requests: write | |
| on: | |
| workflow_call: | |
| inputs: | |
| primaryId: | |
| type: string | |
| required: true | |
| description: Primary Run ID to carry through layers | |
| prerun: | |
| type: string | |
| depth: | |
| type: number | |
| required: true | |
| max-depth: | |
| type: number | |
| required: true | |
| base-branch: | |
| type: string | |
| required: true | |
| prerun: | |
| type: string | |
| secrets: | |
| TESTDRIVER_API_KEY: | |
| required: true | |
| GH_TOKEN: | |
| required: true | |
| LOGIN_USERNAME: | |
| required: false | |
| LOGIN_PASSWORD: | |
| required: false | |
| outputs: | |
| results: | |
| value: ${{ jobs.get-results.outputs.results }} | |
| jobs: | |
| gather-markdown-files: | |
| name: Get new markdown files (./testdriver/generate/*.md) | |
| runs-on: ubuntu-latest | |
| outputs: | |
| markdown_files: ${{ steps.markdown_list.outputs.files }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ inputs.base-branch }} | |
| fetch-depth: 0 | |
| - name: Find all markdown files and extract filenames | |
| id: markdown_list | |
| run: | | |
| # Get list of .md files changed in the last commit | |
| FILES=$(git show --name-only --format= | grep "^testdriver/generate/.*\.md$" || true) | |
| FILENAMES=$(echo "$FILES" | while IFS= read -r file; do basename "$file" .md; done) | |
| FILES_JSON=$(echo "$FILENAMES" | jq -R -s -c 'split("\n")[:-1]') | |
| echo "files=$FILES_JSON" >> $GITHUB_OUTPUT | |
| test: | |
| name: Generate Test - ${{ matrix.markdown }} | |
| needs: gather-markdown-files | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| markdown: ${{ fromJson(needs.gather-markdown-files.outputs.markdown_files) }} | |
| fail-fast: false | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ inputs.base-branch }} | |
| - name: Read file content into environment variable | |
| id: read_file | |
| run: | | |
| CONTENT=$(cat ./testdriver/generate/${{ matrix.markdown }}.md) | |
| echo "MARKDOWN_CONTENT<<EOF" >> $GITHUB_ENV | |
| if [ -f "./testdriver/login.yml" ]; then | |
| echo "0. /run testdriver/login.yml" >> $GITHUB_ENV | |
| fi | |
| echo "$CONTENT" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| - id: generate-filename | |
| name: Generate Random Short Filename | |
| shell: bash | |
| run: | |
| echo "filename=${{ matrix.markdown }}_${{ github.run_id }}_${{ github.run_attempt }}.yml" >> | |
| $GITHUB_OUTPUT | |
| - id: generate-pr-branch | |
| name: Generate PR Branch Name | |
| shell: bash | |
| run: | |
| echo "pr-branch=testdriver/regression-${{ matrix.markdown }}-${{ github.run_id }}-${{ | |
| github.run_attempt }}" >> $GITHUB_OUTPUT | |
| - name: Run AI Regression Test | |
| uses: testdriverai/action@main | |
| continue-on-error: true | |
| with: | |
| version: 4.2.12 | |
| key: ${{ secrets.TESTDRIVER_API_KEY }} | |
| prompt: | | |
| ${{ env.MARKDOWN_CONTENT }} | |
| 1. /summarize | |
| prerun: $${{ inputs.prerun }} | |
| create-pr: true | |
| branch: ${{ inputs.base-branch }} | |
| pr-base: ${{ inputs.base-branch }} | |
| pr-title: "Generated Regression Test: ${{ matrix.markdown }} - Depth: ${{ inputs.depth }}/${{ inputs.max-depth }} [${{ inputs.primaryId }}]" | |
| pr-branch: ${{ steps.generate-pr-branch.outputs.pr-branch }} | |
| pr-test-filename: ${{steps.generate-filename.outputs.filename}} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| FORCE_COLOR: "3" | |
| - id: matrix | |
| run: | | |
| matrix="{\"markdown\": \"${{ matrix.markdown }}\", \"branch\": \"${{ steps.generate-pr-branch.outputs.pr-branch }}\", \"filename\": \"${{ steps.generate-filename.outputs.filename }}\"}" | |
| echo "$matrix" > matrix | |
| echo "matrix=$matrix" >> $GITHUB_OUTPUT | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ hashFiles('matrix') || 'none' }} | |
| path: matrix | |
| if-no-files-found: warn | |
| get-results: | |
| if: ${{inputs.depth != inputs.max-depth}} | |
| name: Get Results | |
| needs: test | |
| runs-on: ubuntu-latest | |
| outputs: | |
| results: ${{ steps.get-results.outputs.results }} | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| - id: get-results | |
| run: | | |
| matrix="$(cat */matrix | jq -c --slurp .)" | |
| echo "results: $matrix" | |
| echo "results=$matrix" >> $GITHUB_OUTPUT | |
| - name: Delete Artifact | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const artifacts = await github.rest.actions.listWorkflowRunArtifacts({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| run_id: ${{ toJson(github.run_id) }} | |
| }).then(r => r.data.artifacts); | |
| console.log({ artifacts }); | |
| for(const artifact of artifacts) { | |
| github.rest.actions.deleteArtifact({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| artifact_id: artifact.id | |
| }); | |
| } |