Outerloop Tests #67
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Executes quarantined tests in the outerloop | |
| name: Outerloop Tests | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 */2 * * *' # Every 2 hours | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| test: | |
| name: ${{ matrix.os.title }} | |
| runs-on: ${{ matrix.os.name }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - name: ubuntu-latest | |
| title: Linux | |
| - name: windows-latest | |
| title: Windows | |
| steps: | |
| - name: Setup vars (Linux) | |
| if: ${{ matrix.os.name == 'ubuntu-latest' }} | |
| run: | | |
| echo "DOTNET_SCRIPT=./dotnet.sh" >> $GITHUB_ENV | |
| echo "BUILD_SCRIPT=./build.sh" >> $GITHUB_ENV | |
| - name: Setup vars (Windows) | |
| if: ${{ matrix.os.name == 'windows-latest' }} | |
| run: | | |
| echo "DOTNET_SCRIPT=.\dotnet.cmd" >> $env:GITHUB_ENV | |
| echo "BUILD_SCRIPT=.\build.cmd" >> $env:GITHUB_ENV | |
| - name: Checkout code | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Trust HTTPS development certificate (Linux) | |
| if: matrix.os.name == 'ubuntu-latest' | |
| run: ${{ env.DOTNET_SCRIPT }} dev-certs https --trust | |
| - name: Run quarantined tests | |
| env: | |
| CI: false | |
| run: | | |
| ${{ env.BUILD_SCRIPT }} -projects ${{ github.workspace }}/tests/Shared/SolutionTests.proj -ci -restore -build -test -c Release /p:RunQuarantinedTests=true /bl:${{ github.workspace }}/artifacts/log/Release/test-quarantined.binlog | |
| - name: Keep only relevant test logs | |
| if: always() | |
| shell: pwsh | |
| run: | | |
| # Define the directory to search for log files | |
| $logDirectory = "${{ github.workspace }}/artifacts/log/**/TestLogs" | |
| # Define the text to search for in the log files | |
| $searchText = "No test matches the given testcase filter" | |
| $resultsFilePattern = "Results File: (.+)" | |
| # Get all .log files in the specified directory and its subdirectories | |
| $logFiles = Get-ChildItem -Path $logDirectory -Filter *.log -Recurse | |
| foreach ($logFile in $logFiles) { | |
| # Read the content of the log file | |
| $content = Get-Content -Path $logFile.FullName | |
| # Check if the content contains the specified text | |
| if ($content -match $searchText) { | |
| # Remove the log file if it contains the specified text | |
| Remove-Item -Path $logFile.FullName -Force | |
| Write-Host "Removed file: $($logFile.FullName)" | |
| } | |
| else { | |
| # Extract paths from lines containing "Results File: <path>" | |
| foreach ($line in $content) { | |
| if ($line -match $resultsFilePattern) { | |
| $resultsFilePath = $matches[1] | |
| Write-Host "Found results file: $resultsFilePath" | |
| # Copy the results file to the TestLogs folder | |
| $destinationPath = (Split-Path -Path $logFile.FullName -Parent) | |
| Copy-Item -Path $resultsFilePath -Destination $destinationPath -Force | |
| Write-Host "Copied $resultsFilePath to $destinationPath" | |
| } | |
| } | |
| } | |
| } | |
| - name: Process logs and post results | |
| if: always() | |
| shell: pwsh | |
| run: | | |
| $logDirectory = "${{ github.workspace }}/artifacts/log/**/TestLogs" | |
| $trxFiles = Get-ChildItem -Path $logDirectory -Filter *.trx -Recurse | |
| $testResults = @() # Initialize an array to store test results | |
| foreach ($trxFile in $trxFiles) { | |
| # Load the .trx file as XML | |
| $xmlContent = [xml](Get-Content -Path $trxFile.FullName) | |
| # Extract test results from the XML | |
| foreach ($testResult in $xmlContent.TestRun.Results.UnitTestResult) { | |
| $testName = $testResult.testName | |
| $outcome = $testResult.outcome | |
| $duration = $testResult.duration | |
| # Map outcome to emoji | |
| switch ($outcome) { | |
| "Passed" { $emoji = "✔️" } | |
| "Failed" { $emoji = "❌" } | |
| default { $emoji = "❔" } | |
| } | |
| # Normalize the duration to a consistent format (hh:mm:ss.fff) | |
| $normalizedDuration = [TimeSpan]::Parse($duration).ToString("mm\:ss\.fff") | |
| # Add the test result to the array | |
| $testResults += [PSCustomObject]@{ | |
| TestName = $testName | |
| Outcome = $outcome | |
| OutcomeIcon = $emoji | |
| Duration = $normalizedDuration | |
| } | |
| } | |
| } | |
| # Sort the test results by test name | |
| $testResults = $testResults | Sort-Object -Property TestName | |
| # Calculate summary statistics | |
| $totalTests = $testResults.Count | |
| $passedTests = ($testResults | Where-Object { $_.Outcome -eq "Passed" }).Count | |
| $failedTests = ($testResults | Where-Object { $_.Outcome -eq "Failed" }).Count | |
| $skippedTests = ($testResults | Where-Object { $_.Outcome -eq "NotExecuted" }).Count | |
| # Add the summary to the annotation | |
| $summary = "total: $totalTests, passed: $passedTests, failed: $failedTests, skipped: $skippedTests" | |
| if ($failedTests -gt 0) { | |
| Write-Host "::error::Tests Summary: $summary" | |
| } else { | |
| Write-Host "::notice::Tests Summary: $summary" | |
| } | |
| # Format the test results as a console-friendly table | |
| $tableHeader = "{0,-16} {1,-150} {2,-20}" -f "Duration", "Test Name", "Result" | |
| $tableSeparator = "-" * 185 | |
| $tableRows = $testResults | ForEach-Object { "{0,-16} {1,-150} {2,-20}" -f $_.Duration, $_.TestName, "$($_.OutcomeIcon) $($_.Outcome)" } | |
| $table = "$tableHeader`n$tableSeparator`n" + ($tableRows -join "`n") + "`n$tableSeparator`n" | |
| Write-Host "`nTest Results:`n`n$table" | |
| # Optionally, save the results to a file for further processing | |
| $outputPath = "${{ github.workspace }}/artifacts/log/Release/TestLogs/summary.log" | |
| $table | Out-File -FilePath $outputPath -Encoding utf8 | |
| Write-Host "Test results saved to $outputPath" | |
| # Windows-specific: Check for failed tests and set the exit code accordingly | |
| # This is a workaround for the issue with the `exit` command in PowerShell | |
| if ($failedTests -gt 0) { | |
| Write-Host "::error::Build failed. Check errors above." | |
| exit 1 | |
| } | |
| - name: Upload logs, and test results | |
| if: always() | |
| uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 | |
| with: | |
| name: logs-${{ matrix.os.name }} | |
| path: | | |
| ${{ github.workspace }}/artifacts/log/*/*.binlog | |
| ${{ github.workspace }}/artifacts/log/*/TestLogs/** | |
| retention-days: 5 |