Fix issues when timeout occurs in test #434
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Test All Skills | |
| # | |
| # Runs the complete test suite for all skills. | |
| # Triggered manually or on significant changes. | |
| name: Test All Skills | |
| permissions: | |
| contents: read | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| skill-name: | |
| description: 'Skill to test (leave empty for all skills)' | |
| required: false | |
| type: string | |
| default: '' | |
| coverage: | |
| description: 'Generate coverage report' | |
| required: false | |
| type: boolean | |
| default: true | |
| run-integration-tests: | |
| description: 'Run integration tests' | |
| type: boolean | |
| default: true | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - 'tests/**' | |
| - 'plugin/skills/**' | |
| - '.github/workflows/test-*.yml' | |
| pull_request: | |
| paths: | |
| - 'tests/**' | |
| - 'plugin/skills/**' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| test: | |
| name: Run All Skill Tests | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: tests | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| cache: 'npm' | |
| cache-dependency-path: tests/package-lock.json | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Run tests | |
| run: | | |
| TEST_ARGS="" | |
| if [ -n "${{ inputs.skill-name }}" ]; then | |
| TEST_ARGS="--testPathPattern=${{ inputs.skill-name }}" | |
| fi | |
| if [ "${{ inputs.coverage }}" = "true" ] || [ "${{ github.event_name }}" = "push" ]; then | |
| npm run test:ci -- --coverage $TEST_ARGS | |
| else | |
| npm run test:ci -- $TEST_ARGS | |
| fi | |
| env: | |
| CI: true | |
| continue-on-error: true | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-results-all | |
| path: tests/reports/junit.xml | |
| retention-days: 30 | |
| - name: Upload coverage report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-all | |
| path: tests/coverage/ | |
| retention-days: 30 | |
| - name: Test Report Summary | |
| if: always() | |
| run: | | |
| echo "## Test Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ -f reports/junit.xml ]; then | |
| # Parse junit.xml for summary | |
| TESTS=$(grep -o 'tests="[0-9]*"' reports/junit.xml | head -1 | grep -o '[0-9]*') | |
| FAILURES=$(grep -o 'failures="[0-9]*"' reports/junit.xml | head -1 | grep -o '[0-9]*') | |
| ERRORS=$(grep -o 'errors="[0-9]*"' reports/junit.xml | head -1 | grep -o '[0-9]*') | |
| echo "| Metric | Count |" >> $GITHUB_STEP_SUMMARY | |
| echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| Tests | $TESTS |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Failures | $FAILURES |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Errors | $ERRORS |" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Coverage Summary | |
| if: always() && (inputs.coverage == true || github.event_name == 'push') | |
| run: | | |
| echo "## Coverage Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ -f coverage/coverage-summary.json ]; then | |
| node -e " | |
| const c = require('./coverage/coverage-summary.json').total; | |
| console.log('| Metric | Coverage |'); | |
| console.log('|--------|----------|'); | |
| console.log('| Statements | ' + c.statements.pct + '% |'); | |
| console.log('| Branches | ' + c.branches.pct + '% |'); | |
| console.log('| Functions | ' + c.functions.pct + '% |'); | |
| console.log('| Lines | ' + c.lines.pct + '% |'); | |
| " >> $GITHUB_STEP_SUMMARY | |
| fi | |
| # COPILOT_CLI_TOKEN must be a fine-grained access token with the Account level Copilot Request permission | |
| # saved to the secrets of the microsoft/GitHub-Copilot-for-Azure repo. | |
| # As a known issue, when the permission of the token doesn't grant sufficient access, | |
| # the agent will hang instead of giving an error. | |
| # If you see the agent hang indefinitely, try refreshing the access token. | |
| # See https://github.com/github/copilot-sdk/issues/343 for more details. | |
| - name: Setup Copilot CLI | |
| id: setup-copilot-cli | |
| if: inputs.run-integration-tests && (github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') | |
| uses: mvkaran/setup-copilot-cli@v1 | |
| with: | |
| token: ${{ secrets.COPILOT_CLI_TOKEN }} | |
| - name: Integration tests | |
| id: integration-tests | |
| if: inputs.run-integration-tests && (github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') | |
| env: | |
| DEBUG: 1 | |
| GH_HEAD_SHA: ${{ github.sha }} | |
| run: | | |
| if [ -n "${{ inputs.skill-name }}" ]; then | |
| TEST_ARGS="--testPathPattern=${{ inputs.skill-name }}" | |
| fi | |
| npm run test:integration -- --testNamePattern="skill-invocation" $TEST_ARGS | |
| # One or more result-{skill}.txt will be written to tests/ | |
| npm run mergeSkillInvocationTestResult | |
| # skill-invocation-test-summary.md will be written to tests/ | |
| cat skill-invocation-test-summary.md | |
| # Read the summary file and save to step output with commit hash | |
| if [ -f skill-invocation-test-summary.md ]; then | |
| echo "skill invocation test summary" >> $GITHUB_STEP_SUMMARY | |
| echo "**Commit:** \`${GH_HEAD_SHA:0:7}\` ($GH_HEAD_SHA)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| cat skill-invocation-test-summary.md >> $GITHUB_STEP_SUMMARY | |
| fi | |
| continue-on-error: true | |
| - name: Upload integration test results | |
| if: inputs.run-integration-tests && (github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-results-skill-invocation | |
| path: tests/skill-invocation-test-summary.md | |
| retention-days: 30 | |
| update-readme: | |
| name: Update Coverage Grid | |
| needs: test | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| permissions: | |
| contents: write | |
| defaults: | |
| run: | |
| working-directory: tests | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| - name: Download coverage artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: coverage-all | |
| path: tests/coverage/ | |
| - name: Generate coverage grid | |
| run: npm run coverage:grid |