Nightly Full Test Suite #166
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly Full Test Suite | |
| on: | |
| schedule: | |
| # Run at 2 AM UTC every day | |
| - cron: '0 2 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| include_hardware_tests: | |
| description: 'Include hardware GPU tests' | |
| required: false | |
| default: 'false' | |
| type: boolean | |
| env: | |
| DOTNET_VERSION: '9.0.x' | |
| DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true | |
| DOTNET_CLI_TELEMETRY_OPTOUT: true | |
| DOTNET_NOLOGO: true | |
| jobs: | |
| # ================================================================ | |
| # UNIT & INTEGRATION TESTS (All platforms) | |
| # ================================================================ | |
| unit-integration-tests: | |
| name: Unit & Integration Tests (${{ matrix.os }}) | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 45 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macos-latest] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup .NET | |
| uses: actions/setup-dotnet@v4 | |
| with: | |
| dotnet-version: ${{ env.DOTNET_VERSION }} | |
| - name: Cache NuGet packages | |
| uses: actions/cache@v5 | |
| with: | |
| path: ~/.nuget/packages | |
| key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj', '**/Directory.*.props') }} | |
| restore-keys: | | |
| ${{ runner.os }}-nuget- | |
| - name: Restore dependencies | |
| run: dotnet restore DotCompute.sln | |
| - name: Build Release | |
| run: dotnet build DotCompute.sln --configuration Release --no-restore | |
| - name: Run Unit & Integration Tests | |
| run: | | |
| dotnet test DotCompute.sln \ | |
| --configuration Release \ | |
| --no-build \ | |
| --filter "Category!=Hardware&Category!=GPU&Category!=CUDA" \ | |
| --logger "trx;LogFilePrefix=nightly-${{ matrix.os }}" \ | |
| --logger "console;verbosity=normal" \ | |
| --collect:"XPlat Code Coverage" \ | |
| --results-directory ./TestResults \ | |
| --blame-hang-timeout 600s | |
| continue-on-error: true | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: test-results-${{ matrix.os }} | |
| path: TestResults/ | |
| retention-days: 30 | |
| # ================================================================ | |
| # HARDWARE TESTS (macOS for Metal) | |
| # ================================================================ | |
| hardware-tests-metal: | |
| name: Hardware Tests (Metal on macOS) | |
| runs-on: macos-latest | |
| timeout-minutes: 60 | |
| if: github.event_name == 'workflow_dispatch' && inputs.include_hardware_tests == 'true' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| - name: Setup .NET | |
| uses: actions/setup-dotnet@v4 | |
| with: | |
| dotnet-version: ${{ env.DOTNET_VERSION }} | |
| - name: Restore dependencies | |
| run: dotnet restore DotCompute.sln | |
| - name: Build Release | |
| run: dotnet build DotCompute.sln --configuration Release --no-restore | |
| - name: Run Metal Hardware Tests | |
| run: | | |
| dotnet test DotCompute.sln \ | |
| --configuration Release \ | |
| --no-build \ | |
| --filter "Category=Metal|Category=Hardware" \ | |
| --logger "trx;LogFilePrefix=hardware-metal" \ | |
| --logger "console;verbosity=detailed" \ | |
| --results-directory ./TestResults/Hardware \ | |
| --blame-hang-timeout 600s | |
| continue-on-error: true | |
| - name: Upload hardware test results | |
| if: always() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: hardware-test-results-metal | |
| path: TestResults/Hardware/ | |
| retention-days: 30 | |
| # ================================================================ | |
| # CODE COVERAGE REPORT | |
| # ================================================================ | |
| coverage-report: | |
| name: Generate Coverage Report | |
| runs-on: ubuntu-latest | |
| needs: [unit-integration-tests] | |
| if: always() | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| - name: Download all test results | |
| uses: actions/download-artifact@v7 | |
| with: | |
| pattern: test-results-* | |
| path: ./AllTestResults | |
| merge-multiple: true | |
| - name: Install report generator | |
| run: dotnet tool install --global dotnet-reportgenerator-globaltool | |
| - name: Generate detailed coverage report | |
| run: | | |
| reportgenerator \ | |
| -reports:AllTestResults/**/coverage.cobertura.xml \ | |
| -targetdir:CoverageReport \ | |
| -reporttypes:Html;Badges;TextSummary;Cobertura \ | |
| -historydir:CoverageHistory | |
| continue-on-error: true | |
| - name: Display coverage summary | |
| run: | | |
| if [ -f CoverageReport/Summary.txt ]; then | |
| cat CoverageReport/Summary.txt | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "## 📊 Coverage Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "\`\`\`" >> $GITHUB_STEP_SUMMARY | |
| cat CoverageReport/Summary.txt >> $GITHUB_STEP_SUMMARY | |
| echo "\`\`\`" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Upload coverage report | |
| if: always() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: coverage-report | |
| path: CoverageReport/ | |
| retention-days: 30 | |
| - name: Upload to Codecov | |
| if: hashFiles('CoverageReport/Cobertura.xml') != '' | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| file: CoverageReport/Cobertura.xml | |
| flags: nightly | |
| name: nightly-coverage | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| continue-on-error: true | |
| # ================================================================ | |
| # TEST SUMMARY & NOTIFICATION | |
| # ================================================================ | |
| test-summary: | |
| name: Test Summary & Notification | |
| runs-on: ubuntu-latest | |
| needs: [unit-integration-tests, coverage-report] | |
| if: always() | |
| steps: | |
| - name: Download all test results | |
| uses: actions/download-artifact@v7 | |
| with: | |
| pattern: test-results-* | |
| path: ./AllTestResults | |
| merge-multiple: true | |
| continue-on-error: true | |
| - name: Publish test summary | |
| uses: dorny/test-reporter@v1 | |
| if: always() | |
| with: | |
| name: Nightly Test Results | |
| path: 'AllTestResults/**/*.trx' | |
| reporter: dotnet-trx | |
| fail-on-error: false | |
| continue-on-error: true | |
| - name: Create test summary | |
| run: | | |
| echo "## 🌙 Nightly Test Suite Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Date:** $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ "${{ needs.unit-integration-tests.result }}" == "success" ]; then | |
| echo "✅ **Unit & Integration Tests:** Passed" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "❌ **Unit & Integration Tests:** Failed or Incomplete" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "📦 **Artifacts:**" >> $GITHUB_STEP_SUMMARY | |
| echo "- Test results and coverage reports are available in the workflow artifacts" >> $GITHUB_STEP_SUMMARY | |
| echo "- Reports are retained for 30 days" >> $GITHUB_STEP_SUMMARY | |
| - name: Create issue on failure | |
| if: failure() && github.event_name == 'schedule' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const date = new Date().toISOString().split('T')[0]; | |
| await github.rest.issues.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| title: `Nightly test failure - ${date}`, | |
| body: `The nightly test suite failed on ${date}.\n\n` + | |
| `**Workflow Run:** ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}\n\n` + | |
| `**Please investigate:**\n` + | |
| `- Review test results in workflow artifacts\n` + | |
| `- Check for infrastructure issues\n` + | |
| `- Review recent commits for potential breakage\n\n` + | |
| `This issue was automatically created by the nightly test workflow.`, | |
| labels: ['test-failure', 'nightly', 'automated'] | |
| }); |