tests-outerloop.yml

# Executes quarantined tests in the outerloop
name: Outerloop Tests

on:
  workflow_dispatch:
  schedule:
    - cron: '0 */2 * * *' # Every 2 hours

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:

  test:
    name: ${{ matrix.os.title }}
    runs-on: ${{ matrix.os.name }}
    strategy:
      fail-fast: false
      matrix:
        os:
          - name: ubuntu-latest
            title: Linux
          - name: windows-latest
            title: Windows
    steps:
      - name: Setup vars (Linux)
        if: ${{ matrix.os.name == 'ubuntu-latest' }}
        run: |
          echo "DOTNET_SCRIPT=./dotnet.sh" >> $GITHUB_ENV
          echo "BUILD_SCRIPT=./build.sh" >> $GITHUB_ENV

      - name: Setup vars (Windows)
        if: ${{ matrix.os.name == 'windows-latest' }}
        run: |
          echo "DOTNET_SCRIPT=.\dotnet.cmd" >> $env:GITHUB_ENV
          echo "BUILD_SCRIPT=.\build.cmd" >> $env:GITHUB_ENV

      - name: Checkout code
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

      - name: Trust HTTPS development certificate (Linux)
        if: matrix.os.name == 'ubuntu-latest'
        run: ${{ env.DOTNET_SCRIPT }} dev-certs https --trust

      - name: Run quarantined tests
        env:
          CI: false
        run: |
          ${{ env.BUILD_SCRIPT }} -projects ${{ github.workspace }}/tests/Shared/SolutionTests.proj -ci -restore -build -test -c Release /p:RunQuarantinedTests=true /bl:${{ github.workspace }}/artifacts/log/Release/test-quarantined.binlog

      - name: Keep only relevant test logs
        if: always()
        shell: pwsh
        run: |
          # Define the directory to search for log files
          $logDirectory = "${{ github.workspace }}/artifacts/log/**/TestLogs"

          # Define the text to search for in the log files
          $searchText = "No test matches the given testcase filter"
          $resultsFilePattern = "Results File: (.+)"

          # Get all .log files in the specified directory and its subdirectories
          $logFiles = Get-ChildItem -Path $logDirectory -Filter *.log -Recurse

          foreach ($logFile in $logFiles) {
              # Read the content of the log file
              $content = Get-Content -Path $logFile.FullName

              # Check if the content contains the specified text
              if ($content -match $searchText) {
                  # Remove the log file if it contains the specified text
                  Remove-Item -Path $logFile.FullName -Force
                  Write-Host "Removed file: $($logFile.FullName)"
              }
              else {
                  # Extract paths from lines containing "Results File: <path>"
                  foreach ($line in $content) {
                      if ($line -match $resultsFilePattern) {
                          $resultsFilePath = $matches[1]
                          Write-Host "Found results file: $resultsFilePath"

                          # Copy the results file to the TestLogs folder
                          $destinationPath = (Split-Path -Path $logFile.FullName -Parent)
                          Copy-Item -Path $resultsFilePath -Destination $destinationPath -Force
                          Write-Host "Copied $resultsFilePath to $destinationPath"
                      }
                  }
              }
          }

      - name: Process logs and post results
        if: always()
        shell: pwsh
        run: |
          $logDirectory = "${{ github.workspace }}/artifacts/log/**/TestLogs"
          $trxFiles = Get-ChildItem -Path $logDirectory -Filter *.trx -Recurse

          $testResults = @() # Initialize an array to store test results

          foreach ($trxFile in $trxFiles) {
              # Load the .trx file as XML
              $xmlContent = [xml](Get-Content -Path $trxFile.FullName)

              # Extract test results from the XML
              foreach ($testResult in $xmlContent.TestRun.Results.UnitTestResult) {
                  $testName = $testResult.testName
                  $outcome = $testResult.outcome
                  $duration = $testResult.duration

                  # Map outcome to emoji
                  switch ($outcome) {
                      "Passed" { $emoji = "✔️" }
                      "Failed" { $emoji = "❌" }
                      default { $emoji = "❔" }
                  }

                  # Normalize the duration to a consistent format (hh:mm:ss.fff)
                  $normalizedDuration = [TimeSpan]::Parse($duration).ToString("mm\:ss\.fff")

                  # Add the test result to the array
                  $testResults += [PSCustomObject]@{
                      TestName    = $testName
                      Outcome     = $outcome
                      OutcomeIcon = $emoji
                      Duration    = $normalizedDuration
                  }
              }
          }

          # Sort the test results by test name
          $testResults = $testResults | Sort-Object -Property TestName

          # Calculate summary statistics
          $totalTests = $testResults.Count
          $passedTests = ($testResults | Where-Object { $_.Outcome -eq "Passed" }).Count
          $failedTests = ($testResults | Where-Object { $_.Outcome -eq "Failed" }).Count
          $skippedTests = ($testResults | Where-Object { $_.Outcome -eq "NotExecuted" }).Count

          # Add the summary to the annotation
          $summary = "total: $totalTests, passed: $passedTests, failed: $failedTests, skipped: $skippedTests"
          if ($failedTests -gt 0) {
              Write-Host "::error::Tests Summary: $summary"
          } else {
              Write-Host "::notice::Tests Summary: $summary"
          }

          # Format the test results as a console-friendly table
          $tableHeader = "{0,-16} {1,-150} {2,-20}" -f "Duration", "Test Name", "Result"
          $tableSeparator = "-" * 185
          $tableRows = $testResults | ForEach-Object { "{0,-16} {1,-150} {2,-20}" -f $_.Duration, $_.TestName, "$($_.OutcomeIcon) $($_.Outcome)" }
          $table = "$tableHeader`n$tableSeparator`n" + ($tableRows -join "`n") + "`n$tableSeparator`n"
          Write-Host "`nTest Results:`n`n$table"

          # Optionally, save the results to a file for further processing
          $outputPath = "${{ github.workspace }}/artifacts/log/Release/TestLogs/summary.log"
          $table | Out-File -FilePath $outputPath -Encoding utf8
          Write-Host "Test results saved to $outputPath"

          # Windows-specific: Check for failed tests and set the exit code accordingly
          # This is a workaround for the issue with the `exit` command in PowerShell
          if ($failedTests -gt 0) {
              Write-Host "::error::Build failed. Check errors above."
              exit 1
          }

      - name: Upload logs, and test results
        if: always()
        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
        with:
          name: logs-${{ matrix.os.name }}
          path: |
            ${{ github.workspace }}/artifacts/log/*/*.binlog
            ${{ github.workspace }}/artifacts/log/*/TestLogs/**
          retention-days: 5