diff --git a/.claude/skills/quarantine-test/SKILL.md b/.claude/skills/quarantine-test/SKILL.md new file mode 100644 index 00000000000..c2f9588d791 --- /dev/null +++ b/.claude/skills/quarantine-test/SKILL.md @@ -0,0 +1,228 @@ +--- +name: Quarantine Test +description: Quarantine flaky or failing tests by adding the QuarantinedTest attribute using the QuarantineTools. Use when tests are failing intermittently or need to be excluded from regular test runs. +allowed-tools: Read, Grep, Bash +--- + +# Quarantine Test + +## Purpose + +This skill quarantines flaky or failing tests by using the QuarantineTools utility to add the `[QuarantinedTest]` attribute. Quarantined tests are excluded from regular test runs and instead run in a separate quarantine workflow. + +## When to Use + +Invoke this skill when: +- A test is failing intermittently (flaky) +- A test doesn't fail deterministically +- User requests to "quarantine a test" +- A test needs to be temporarily excluded from CI while being fixed +- You need to mark a test as quarantined with an associated GitHub issue + +## Important Context + +- Quarantined tests are marked with `[QuarantinedTest("issue-url")]` attribute +- They are NOT run in the regular `tests.yml` workflow +- They ARE run in the separate `tests-quarantine.yml` workflow every 6 hours +- A GitHub issue URL is REQUIRED when quarantining tests +- The QuarantineTools utility handles adding the attribute and managing using directives automatically + +## Instructions + +### Step 1: Identify the Test to Quarantine + +1. Get the fully-qualified test method name in format: `Namespace.ClassName.TestMethodName` +2. If user provides partial name, use Grep to find the complete qualified name: + ```bash + grep -rn "void TestMethodName" tests/ + ``` +3. Extract the namespace and class name from the file to build the fully-qualified name + +Example: `Aspire.Hosting.Tests.DistributedApplicationTests.TestMethodName` + +### Step 2: Get or Create GitHub Issue + +1. Check if the user provided a GitHub issue URL +2. If not, ask: "What is the GitHub issue URL for tracking this flaky test?" +3. The URL must be a valid http/https URL (e.g., `https://github.com/dotnet/aspire/issues/1234`) +4. If no issue exists, suggest creating one first to track the test failure + +### Step 3: Run QuarantineTools + +Execute the QuarantineTools with the quarantine flag, test name(s), and issue URL: + +```bash +dotnet run --project tools/QuarantineTools/QuarantineTools.csproj -- --quarantine "Namespace.ClassName.TestMethodName" --url "https://github.com/org/repo/issues/1234" +``` + +**Multiple tests** can be quarantined at once: +```bash +dotnet run --project tools/QuarantineTools/QuarantineTools.csproj -- --quarantine "Namespace.Class.Test1" "Namespace.Class.Test2" --url "https://github.com/org/repo/issues/1234" +``` + +**Command line flags:** +- `-q` or `--quarantine`: Quarantine mode (add attribute) +- `-i` or `--url`: GitHub issue URL (required for quarantine) +- Tests: Fully-qualified method names (space-separated) + +### Step 4: Verify the Changes + +1. The tool will output which files were modified: + ``` + Updated 1 file(s): + - tests/ProjectName.Tests/TestFile.cs + ``` + +2. Read the modified file to confirm the attribute was added correctly: + ```bash + grep -A 2 -B 2 "QuarantinedTest" tests/ProjectName.Tests/TestFile.cs + ``` + +3. Verify that: + - The `[QuarantinedTest("issue-url")]` attribute appears above the test method + - The `using Aspire.TestUtilities;` directive was added to the file (if not already present) + +### Step 5: Build and Run Tests to Confirm + +1. Build the test project to ensure no compilation errors: + ```bash + dotnet build tests/ProjectName.Tests/ProjectName.Tests.csproj + ``` + +2. Run the test project with quarantine filter to verify the test is now quarantined: + ```bash + dotnet test tests/ProjectName.Tests/ProjectName.Tests.csproj --no-build -- --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true" + ``` + +3. Confirm the quarantined test is NOT executed in the regular test run + +4. Optionally, verify the test CAN be run with the quarantine filter: + ```bash + dotnet test tests/ProjectName.Tests/ProjectName.Tests.csproj --no-build -- --filter-trait "quarantined=true" + ``` + +### Step 6: Report Results + +Provide a clear summary: +- Which test(s) were quarantined +- The GitHub issue URL used +- Which file(s) were modified +- Confirmation that the test builds and is properly excluded from regular runs +- Remind the user to commit the changes + +## Examples + +### Example 1: Quarantine a single flaky test + +User: "Quarantine the TestDistributedApplicationLifecycle test, it's flaky. Issue: https://github.com/dotnet/aspire/issues/5678" + +Actions: +1. Find the fully-qualified name using Grep: + ```bash + grep -rn "void TestDistributedApplicationLifecycle" tests/ + ``` + Result: `tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs` + +2. Determine qualified name: `Aspire.Hosting.Tests.DistributedApplicationTests.TestDistributedApplicationLifecycle` + +3. Run QuarantineTools: + ```bash + dotnet run --project tools/QuarantineTools/QuarantineTools.csproj -- -q "Aspire.Hosting.Tests.DistributedApplicationTests.TestDistributedApplicationLifecycle" -i "https://github.com/dotnet/aspire/issues/5678" + ``` + +4. Verify output shows file was updated + +5. Build and test: + ```bash + dotnet build tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj + dotnet test tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj --no-build -- --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true" + ``` + +6. Report: "Quarantined test `TestDistributedApplicationLifecycle` with issue #5678. The test is now excluded from regular CI runs and will run in the quarantine workflow." + +### Example 2: Quarantine multiple related tests + +User: "These three tests in RedisTests are all flaky, quarantine them: TestRedisConnection, TestRedisCache, TestRedisCommands. Issue: https://github.com/dotnet/aspire/issues/9999" + +Actions: +1. Find fully-qualified names (assuming they're in `Aspire.Components.Tests.RedisTests`) + +2. Run QuarantineTools with multiple tests: + ```bash + dotnet run --project tools/QuarantineTools/QuarantineTools.csproj -- -q "Aspire.Components.Tests.RedisTests.TestRedisConnection" "Aspire.Components.Tests.RedisTests.TestRedisCache" "Aspire.Components.Tests.RedisTests.TestRedisCommands" -i "https://github.com/dotnet/aspire/issues/9999" + ``` + +3. Verify all three were modified + +4. Build and test the project + +5. Report: "Quarantined 3 tests from RedisTests with issue #9999." + +### Example 3: User provides short test name + +User: "Quarantine CanStartDashboard - it keeps timing out" + +Actions: +1. Ask for GitHub issue: "What is the GitHub issue URL for tracking this flaky test?" + +2. User provides: "https://github.com/dotnet/aspire/issues/4321" + +3. Find the test: + ```bash + grep -rn "void CanStartDashboard" tests/ + ``` + Found in: `tests/Aspire.Dashboard.Tests/DashboardTests.cs` + +4. Read the file to determine namespace: `Aspire.Dashboard.Tests` + +5. Build fully-qualified name: `Aspire.Dashboard.Tests.DashboardTests.CanStartDashboard` + +6. Run QuarantineTools: + ```bash + dotnet run --project tools/QuarantineTools/QuarantineTools.csproj -- -q "Aspire.Dashboard.Tests.DashboardTests.CanStartDashboard" -i "https://github.com/dotnet/aspire/issues/4321" + ``` + +7. Build and verify + +8. Report success + +## Common Issues and Troubleshooting + +### Issue: "No method found matching" +- **Cause**: The fully-qualified name is incorrect +- **Solution**: Use Grep to find the exact namespace, class name, and method name + +### Issue: "The test is already quarantined" +- **Cause**: The attribute already exists on the method +- **Solution**: Verify by reading the test file; no action needed + +### Issue: Tool reports "Quarantining requires a valid http(s) URL" +- **Cause**: The issue URL is missing or malformed +- **Solution**: Ensure the URL starts with `http://` or `https://` + +### Issue: Build fails after quarantining +- **Cause**: The QuarantineTools may have encountered a syntax issue (rare) +- **Solution**: Read the modified file and check for syntax errors; the tool should handle this correctly + +## Important Notes + +1. **Always build after quarantining** to ensure the changes are valid +2. **Run tests to confirm** the quarantined test is properly excluded +3. **Don't forget to commit** the modified test files +4. **Track with GitHub issues** - every quarantined test should have an associated issue +5. **The QuarantineTools handles**: + - Adding the `[QuarantinedTest("url")]` attribute + - Adding `using Aspire.TestUtilities;` if needed + - Preserving file formatting and indentation + - Supporting nested classes and various namespace styles + +## Unquarantining Tests + +To unquarantine a test (remove the attribute), use: +```bash +dotnet run --project tools/QuarantineTools/QuarantineTools.csproj -- -u "Namespace.ClassName.TestMethodName" +``` + +The tool will: +- Remove the `[QuarantinedTest]` attribute +- Remove the `using Aspire.TestUtilities;` directive if no other tests in the file use it diff --git a/.claude/skills/test-runner/SKILL.md b/.claude/skills/test-runner/SKILL.md new file mode 100644 index 00000000000..9a10437ecce --- /dev/null +++ b/.claude/skills/test-runner/SKILL.md @@ -0,0 +1,166 @@ +--- +name: Aspire Test Runner +description: Run tests for the Aspire project correctly, excluding quarantined and outerloop tests, with proper build verification. Use when running tests, debugging test failures, or validating changes. +allowed-tools: Read, Grep, Glob, Bash +--- + +# Aspire Test Runner + +## Purpose + +This skill ensures tests are run correctly in the Aspire repository, following the project's specific requirements for test execution, including proper exclusion of quarantined and outerloop tests. + +## When to Use + +Invoke this skill when: +- Running tests for a specific project or test class +- Debugging test failures +- Validating code changes +- Verifying builds after modifications +- User requests to "run tests" or "test my changes" + +## Critical Requirements + +**ALWAYS exclude quarantined and outerloop tests** in automated environments: +- Quarantined tests are marked with `[QuarantinedTest]` and are known to be flaky +- Outerloop tests are marked with `[OuterloopTest]` and are long-running or resource-intensive +- These tests run separately in dedicated CI workflows + +## Instructions + +### Step 1: Identify Test Target + +1. If the user specifies a test project, use that path +2. If the user mentions specific test methods or classes, identify the containing test project +3. Use Glob to find test projects if needed: + ```bash + find tests -name "*.Tests.csproj" -type f + ``` + +### Step 2: Build Verification (if needed) + +**Important**: Only build if: +- There have been code changes since the last build +- The user hasn't just run a successful build +- You're unsure if the code is up to date + +If building is needed: +```bash +# Quick build with skip native (saves 1-2 minutes) +./build.sh --build /p:SkipNativeBuild=true +``` + +### Step 3: Run Tests with Proper Filters + +**Default test run** (all tests in a project): +```bash +dotnet test tests/{ProjectName}.Tests/{ProjectName}.Tests.csproj --no-build -- --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true" +``` + +**Specific test method**: +```bash +dotnet test tests/{ProjectName}.Tests/{ProjectName}.Tests.csproj --no-build -- --filter-method "*.{MethodName}" --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true" +``` + +**Specific test class**: +```bash +dotnet test tests/{ProjectName}.Tests/{ProjectName}.Tests.csproj --no-build -- --filter-class "*.{ClassName}" --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true" +``` + +**Multiple test methods**: +```bash +dotnet test tests/{ProjectName}.Tests/{ProjectName}.Tests.csproj --no-build -- --filter-method "*.Method1" --filter-method "*.Method2" --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true" +``` + +### Step 4: Handle Test Results + +**If tests pass**: +- Report success to the user +- Mention the number of tests that passed + +**If tests fail**: +1. Analyze the failure output +2. Identify which tests failed and why +3. Check if failures are related to recent code changes +4. Suggest fixes or next steps +5. Do NOT mark the task as complete if tests are failing + +**If snapshot tests fail**: +1. Tests using Verify library will show snapshot differences +2. After verifying the new output is correct, run: + ```bash + dotnet verify accept -y + ``` +3. Re-run the tests to confirm they pass + +### Step 5: Report Results + +Provide a clear summary: +- Number of tests run +- Pass/fail status +- Any warnings or issues +- Next steps if failures occurred + +## Examples + +### Example 1: Run all tests for a specific project + +User: "Run tests for Aspire.Hosting.Testing" + +Actions: +1. Identify test project: `tests/Aspire.Hosting.Testing.Tests/Aspire.Hosting.Testing.Tests.csproj` +2. Run with proper filters: + ```bash + dotnet test tests/Aspire.Hosting.Testing.Tests/Aspire.Hosting.Testing.Tests.csproj --no-build -- --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true" + ``` +3. Report results + +### Example 2: Run a specific test method + +User: "Run the TestingBuilderHasAllPropertiesFromRealBuilder test" + +Actions: +1. Find the test using Grep: + ```bash + grep -r "TestingBuilderHasAllPropertiesFromRealBuilder" tests/ + ``` +2. Identify project: `Aspire.Hosting.Testing.Tests` +3. Run specific test: + ```bash + dotnet test tests/Aspire.Hosting.Testing.Tests/Aspire.Hosting.Testing.Tests.csproj --no-build -- --filter-method "*.TestingBuilderHasAllPropertiesFromRealBuilder" --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true" + ``` +4. Report results + +### Example 3: Run tests after making changes + +User: "I just modified the hosting code, run tests to verify" + +Actions: +1. Identify affected test projects (e.g., `Aspire.Hosting.Tests`) +2. Build first since code was modified: + ```bash + ./build.sh --build /p:SkipNativeBuild=true + ``` +3. Run tests: + ```bash + dotnet test tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj -- --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true" + ``` + Note: No `--no-build` flag since we need to pick up the changes +4. Report results + +## Common Pitfalls to Avoid + +1. **Never omit the quarantine and outerloop filters** - this will run flaky tests +2. **Don't use `--no-build`** if code has changed - the changes won't be tested +3. **Don't run the full test suite** - it takes 30+ minutes, use targeted testing +4. **Don't ignore snapshot test failures** - they indicate output changes that need review +5. **Don't forget the `--` separator** before filter arguments + +## Valid Test Filter Switches + +- `--filter-class` / `--filter-not-class`: Filter by class name +- `--filter-method` / `--filter-not-method`: Filter by method name +- `--filter-namespace` / `--filter-not-namespace`: Filter by namespace +- `--filter-trait` / `--filter-not-trait`: Filter by trait (category, platform, etc.) + +Switches can be repeated to filter multiple values. Class and method filters expect fully qualified names, unless using a prefix like `*.ClassName`. diff --git a/.github/actions/enumerate-tests/action.yml b/.github/actions/enumerate-tests/action.yml index 3db315b0f17..d84fae85e61 100644 --- a/.github/actions/enumerate-tests/action.yml +++ b/.github/actions/enumerate-tests/action.yml @@ -11,12 +11,12 @@ inputs: default: false outputs: - integrations_tests_matrix: - description: Integration tests matrix - value: ${{ steps.generate_integrations_matrix.outputs.integrations_tests_matrix }} - templates_tests_matrix: - description: Templates tests matrix - value: ${{ steps.generate_templates_matrix.outputs.templates_tests_matrix }} + tests_matrix_requires_nugets: + description: Combined tests matrix for tests that require nugets + value: ${{ steps.generate_combined_matrix.outputs.tests_matrix_requires_nugets }} + tests_matrix_no_nugets: + description: Combined tests matrix for tests that do not require nugets + value: ${{ steps.generate_combined_matrix.outputs.tests_matrix_no_nugets }} runs: using: "composite" steps: @@ -28,60 +28,114 @@ runs: with: global-json-file: ${{ github.workspace }}/global.json - - name: Get list of integration tests - if: ${{ inputs.includeIntegrations }} - shell: pwsh - run: > - dotnet build ${{ github.workspace }}/tests/Shared/GetTestProjects.proj - /bl:${{ github.workspace }}/artifacts/log/Debug/GetTestProjects.binlog - /p:TestsListOutputPath=${{ github.workspace }}/artifacts/TestsForGithubActions.list - /p:ContinuousIntegrationBuild=true + - name: Restore + shell: bash + run: ./restore.sh - - name: Generate list of template tests - if: ${{ inputs.includeTemplates }} - shell: pwsh + - name: Generate combined test matrix + shell: bash run: > - dotnet build ${{ github.workspace }}/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj - "/t:Build;ExtractTestClassNames" - /bl:${{ github.workspace }}/artifacts/log/Debug/BuildTemplatesTests.binlog - -p:ExtractTestClassNamesForHelix=true - -p:PrepareForHelix=true - -p:ExtractTestClassNamesPrefix=Aspire.Templates.Tests - -p:InstallBrowsersForPlaywright=false - - - name: Generate tests matrix - id: generate_integrations_matrix - if: ${{ inputs.includeIntegrations }} - shell: pwsh - run: | - $filePath = "${{ github.workspace }}/artifacts/TestsForGithubActions.list" - $lines = Get-Content $filePath - $jsonObject = @{ - "shortname" = $lines | Sort-Object - } - $jsonString = ConvertTo-Json $jsonObject -Compress - "integrations_tests_matrix=$jsonString" - "integrations_tests_matrix=$jsonString" | Out-File -FilePath $env:GITHUB_OUTPUT + ./build.sh -test + /p:TestRunnerName=TestEnumerationRunsheetBuilder + /p:TestsListOutputPath=artifacts/TestsForGithubActions.list + /p:TestMatrixOutputPath=artifacts/combined-test-matrix.json + /p:BuildOs=${{ runner.os == 'Linux' && 'linux' || runner.os == 'macOS' && 'darwin' || 'windows' }} + -p:GenerateCIPartitions=true + -bl - - name: Generate templates matrix - id: generate_templates_matrix - if: ${{ inputs.includeTemplates }} + - name: Generate combined matrix outputs + id: generate_combined_matrix shell: pwsh run: | - $inputFilePath = "${{ github.workspace }}/artifacts/helix/templates-tests/Aspire.Templates.Tests.tests.list" - $lines = Get-Content $inputFilePath + $matrixFilePath = "${{ github.workspace }}/artifacts/combined-test-matrix.json" + if (Test-Path $matrixFilePath) { + $matrixContent = Get-Content -Raw $matrixFilePath | ConvertFrom-Json - $prefix = "Aspire.Templates.Tests." - $lines = Get-Content $inputFilePath | ForEach-Object { - $_ -replace "^$prefix", "" - } + # Define defaults to apply when properties are missing + $defaults = @{ + extraTestArgs = '' + requiresNugets = $false + requiresTestSdk = $false + testSessionTimeout = '20m' + testHangTimeout = '10m' + supportedOSes = @('windows', 'linux', 'macos') + } + + # Split tests based on requiresNugets property and expand by OS + $testsRequiringNugets = @() + $testsNotRequiringNugets = @() + + foreach ($test in $matrixContent.include) { + # Apply defaults for missing properties + foreach ($key in $defaults.Keys) { + if ($key -ne 'supportedOSes' -and -not $test.PSObject.Properties.Name.Contains($key)) { + $test | Add-Member -NotePropertyName $key -NotePropertyValue $defaults[$key] -Force + } + } + + # Get supported OSes (use test-specific if present, otherwise default) + $supportedOSes = if ($test.PSObject.Properties.Name.Contains('supportedOSes')) { + $test.supportedOSes + } else { + $defaults.supportedOSes + } + + # Expand test entry for each supported OS + foreach ($os in $supportedOSes) { + # Create a new entry for each OS + $testCopy = [PSCustomObject]@{} + foreach ($prop in $test.PSObject.Properties) { + if ($prop.Name -ne 'supportedOSes') { + $testCopy | Add-Member -NotePropertyName $prop.Name -NotePropertyValue $prop.Value -Force + } + } + + # Add OS runner mapping + $osRunner = switch ($os) { + 'windows' { 'windows-latest' } + 'linux' { 'ubuntu-latest' } + 'macos' { 'macos-latest' } + default { 'ubuntu-latest' } + } + $testCopy | Add-Member -NotePropertyName 'runs-on' -NotePropertyValue $osRunner -Force + + # Normalize boolean values to actual booleans (not strings) + if ($testCopy.PSObject.Properties.Name.Contains('requiresNugets')) { + $testCopy.requiresNugets = ($testCopy.requiresNugets -eq 'true' -or $testCopy.requiresNugets -eq $true) + } + if ($testCopy.PSObject.Properties.Name.Contains('requiresTestSdk')) { + $testCopy.requiresTestSdk = ($testCopy.requiresTestSdk -eq 'true' -or $testCopy.requiresTestSdk -eq $true) + } + + # Add to appropriate list based on requiresNugets + if ($testCopy.requiresNugets -eq $true) { + $testsRequiringNugets += $testCopy + } else { + $testsNotRequiringNugets += $testCopy + } + } + } + + # Create matrices + $nugetMatrix = @{ "include" = $testsRequiringNugets } + $nonNugetMatrix = @{ "include" = $testsNotRequiringNugets } + + $nugetMatrixJson = ConvertTo-Json $nugetMatrix -Compress -Depth 10 + $nonNugetMatrixJson = ConvertTo-Json $nonNugetMatrix -Compress -Depth 10 + + "tests_matrix_requires_nugets=$nugetMatrixJson" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + "tests_matrix_no_nugets=$nonNugetMatrixJson" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - $jsonObject = @{ - "shortname" = $lines | Sort-Object + Write-Host "Combined test matrices generated successfully" + Write-Host "Tests requiring nugets: $($testsRequiringNugets.Count)" + Write-Host "Tests not requiring nugets: $($testsNotRequiringNugets.Count)" + } else { + # Empty matrices if no combined matrix found + $emptyMatrix = '{"include":[]}' + "tests_matrix_requires_nugets=$emptyMatrix" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + "tests_matrix_no_nugets=$emptyMatrix" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + Write-Host "No combined test matrix found, using empty matrices" } - $jsonString = ConvertTo-Json $jsonObject -Compress - "templates_tests_matrix=$jsonString" - "templates_tests_matrix=$jsonString" | Out-File -FilePath $env:GITHUB_OUTPUT - name: Upload logs if: always() @@ -91,3 +145,4 @@ runs: path: | artifacts/log/**/*.binlog artifacts/**/*.list + artifacts/combined-test-matrix.json diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 3c366b56c41..e78871c1f7d 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -156,17 +156,6 @@ These switches can be repeated to run tests on multiple classes or methods at on 2. **Package Generation**: `./build.sh --pack` verifies all packages can be created 3. **Specific Tests**: Target individual test projects related to your changes -## Quarantined tests - -- Tests that are flaky and don't fail deterministically are marked with the `QuarantinedTest` attribute. -- Such tests are not run as part of the regular tests workflow (`tests.yml`). - - Instead they are run in the `Quarantine` workflow (`tests-quarantine.yml`). -- A github issue url is used with the attribute - -Example: `[QuarantinedTest("..issue url..")]` - -- To quarantine or unquarantine tests, use the tool in `tools/QuarantineTools/QuarantineTools.csproj`. - ## Outerloop tests - Tests that are long-running, resource-intensive, or require special infrastructure are marked with the `OuterloopTest` attribute. @@ -194,8 +183,6 @@ The `*.Designer.cs` files are in the repo, but are intended to match same named * Code blocks should be formatted with triple backticks (```) and include the language identifier for syntax highlighting. * JSON code blocks should be indented properly. -## Localization files -* Files matching the pattern `*/localize/templatestrings.*.json` are localization files. Do not translate their content. It is done by a dedicated workflow. ## Trust These Instructions These instructions are comprehensive and tested. Only search for additional information if: diff --git a/.github/instructions/quarantine.instructions.md b/.github/instructions/quarantine.instructions.md deleted file mode 100644 index 4d4952350ce..00000000000 --- a/.github/instructions/quarantine.instructions.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -applyTo: "tools/QuarantineTools/*" ---- - -This tool is used to quarantine flaky tests. - -Usage: - -```bash -dotnet run --project tools/QuarantineTools -- -q Namespace.Type.Method -i https://issue.url -``` - -Make sure to build the project containing the updated tests to ensure the changes don't break the build. diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a4cd05fa553..88cf2764ca1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,45 +9,12 @@ on: type: string jobs: - # Duplicated jobs so their dependencies are not blocked on both the - # setup jobs - - setup_for_tests_lin: - name: Setup for tests (Linux) + setup_for_tests: + name: Setup for tests runs-on: ubuntu-latest outputs: - integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }} - templates_tests_matrix: ${{ steps.generate_tests_matrix.outputs.templates_tests_matrix }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - uses: ./.github/actions/enumerate-tests - id: generate_tests_matrix - with: - includeIntegrations: true - includeTemplates: true - - setup_for_tests_macos: - name: Setup for tests (macOS) - runs-on: macos-latest - outputs: - integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }} - templates_tests_matrix: ${{ steps.generate_tests_matrix.outputs.templates_tests_matrix }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - uses: ./.github/actions/enumerate-tests - id: generate_tests_matrix - with: - includeIntegrations: true - includeTemplates: true - - setup_for_tests_win: - name: Setup for tests (Windows) - runs-on: windows-latest - outputs: - integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }} - templates_tests_matrix: ${{ steps.generate_tests_matrix.outputs.templates_tests_matrix }} + tests_matrix_requires_nugets: ${{ steps.generate_tests_matrix.outputs.tests_matrix_requires_nugets }} + tests_matrix_no_nugets: ${{ steps.generate_tests_matrix.outputs.tests_matrix_no_nugets }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -63,115 +30,43 @@ jobs: with: versionOverrideArg: ${{ inputs.versionOverrideArg }} - integrations_test_lin: - uses: ./.github/workflows/run-tests.yml - name: Integrations Linux - needs: setup_for_tests_lin - strategy: - fail-fast: false - matrix: - ${{ fromJson(needs.setup_for_tests_lin.outputs.integrations_tests_matrix) }} - with: - testShortName: ${{ matrix.shortname }} - os: "ubuntu-latest" - # Docker tests are run on linux, and Hosting tests take longer to finish - testSessionTimeout: ${{ matrix.shortname == 'Hosting' && '25m' || '15m' }} - extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\"" - versionOverrideArg: ${{ inputs.versionOverrideArg }} - - integrations_test_macos: - uses: ./.github/workflows/run-tests.yml - name: Integrations macos - needs: setup_for_tests_macos - strategy: - fail-fast: false - matrix: - ${{ fromJson(needs.setup_for_tests_macos.outputs.integrations_tests_matrix) }} - with: - testShortName: ${{ matrix.shortname }} - os: "macos-latest" - extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\"" - versionOverrideArg: ${{ inputs.versionOverrideArg }} - - integrations_test_win: + tests_no_nugets: uses: ./.github/workflows/run-tests.yml - name: Integrations Windows - needs: setup_for_tests_win + name: Tests + needs: setup_for_tests + if: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets).include[0] != null }} strategy: fail-fast: false - matrix: - ${{ fromJson(needs.setup_for_tests_win.outputs.integrations_tests_matrix) }} + matrix: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets) }} with: testShortName: ${{ matrix.shortname }} - os: "windows-latest" - extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\"" + os: ${{ matrix.runs-on }} + testProjectPath: ${{ matrix.testProjectPath }} + testSessionTimeout: ${{ matrix.testSessionTimeout }} + testHangTimeout: ${{ matrix.testHangTimeout }} + extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} + requiresNugets: ${{ matrix.requiresNugets }} + requiresTestSdk: ${{ matrix.requiresTestSdk }} - templates_test_lin: - name: Templates Linux + tests_requires_nugets: + name: Tests (Requires Nugets) uses: ./.github/workflows/run-tests.yml - needs: [setup_for_tests_lin, build_packages] + needs: [setup_for_tests, build_packages] + if: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets).include[0] != null }} strategy: fail-fast: false - matrix: ${{ fromJson(needs.setup_for_tests_lin.outputs.templates_tests_matrix) }} + matrix: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets) }} with: testShortName: ${{ matrix.shortname }} - os: "ubuntu-latest" - testProjectPath: tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj - testSessionTimeout: 20m - testHangTimeout: 12m - extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true --filter-class Aspire.Templates.Tests.${{ matrix.shortname }}" - versionOverrideArg: ${{ inputs.versionOverrideArg }} - requiresNugets: true - requiresTestSdk: true - - templates_test_macos: - name: Templates macos - uses: ./.github/workflows/run-tests.yml - needs: [setup_for_tests_macos, build_packages] - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.setup_for_tests_macos.outputs.templates_tests_matrix) }} - with: - testShortName: ${{ matrix.shortname }} - os: "macos-latest" - testProjectPath: tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj - testSessionTimeout: 20m - testHangTimeout: 12m - extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true --filter-class Aspire.Templates.Tests.${{ matrix.shortname }}" - versionOverrideArg: ${{ inputs.versionOverrideArg }} - requiresNugets: true - requiresTestSdk: true - - templates_test_win: - name: Templates Windows - uses: ./.github/workflows/run-tests.yml - needs: [setup_for_tests_win, build_packages] - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.setup_for_tests_win.outputs.templates_tests_matrix) }} - with: - testShortName: ${{ matrix.shortname }} - os: "windows-latest" - testProjectPath: tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj - testSessionTimeout: 20m - testHangTimeout: 12m - extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true --filter-class Aspire.Templates.Tests.${{ matrix.shortname }}" - versionOverrideArg: ${{ inputs.versionOverrideArg }} - requiresNugets: true - requiresTestSdk: true - - endtoend_tests: - name: EndToEnd Linux - uses: ./.github/workflows/run-tests.yml - needs: build_packages - with: - testShortName: EndToEnd - # EndToEnd is not run on Windows/macOS due to missing Docker support - os: ubuntu-latest - testProjectPath: tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj - requiresNugets: true + os: ${{ matrix.runs-on }} + testProjectPath: ${{ matrix.testProjectPath }} + testSessionTimeout: ${{ matrix.testSessionTimeout }} + testHangTimeout: ${{ matrix.testHangTimeout }} + extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} + requiresNugets: ${{ matrix.requiresNugets }} + requiresTestSdk: ${{ matrix.requiresTestSdk }} extension_tests_win: name: Run VS Code extension tests (Windows) @@ -185,7 +80,7 @@ jobs: - name: Setup Node.js environment uses: actions/setup-node@v2 with: - node-version: ${{ matrix.node-version }} + node-version: '20' - name: Install dependencies run: yarn install - name: Run tests @@ -203,14 +98,9 @@ jobs: runs-on: ubuntu-latest name: Final Test Results needs: [ - endtoend_tests, extension_tests_win, - integrations_test_lin, - integrations_test_macos, - integrations_test_win, - templates_test_lin, - templates_test_macos, - templates_test_win + tests_no_nugets, + tests_requires_nugets ] steps: - name: Checkout code diff --git a/eng/AfterSolutionBuild.targets b/eng/AfterSolutionBuild.targets index 3288a591e3d..c391b2fa6ad 100644 --- a/eng/AfterSolutionBuild.targets +++ b/eng/AfterSolutionBuild.targets @@ -105,4 +105,71 @@ + + + + + + + + + <_TestsListOutputPath Condition="'$(TestsListOutputPath)' != ''">$([MSBuild]::NormalizePath('$(RepoRoot)', '$(TestsListOutputPath)')) + <_TestsListOutputPath Condition="'$(TestsListOutputPath)' == ''">$(ArtifactsDir)/TestsForGithubActions.list + <_TestMatrixOutputPath Condition="'$(TestMatrixOutputPath)' != ''">$([MSBuild]::NormalizePath('$(RepoRoot)', '$(TestMatrixOutputPath)')) + <_BuildMatrixScript>$([MSBuild]::NormalizePath($(RepoRoot), 'eng', 'scripts', 'build-test-matrix.ps1')) + + + <_HelixDir>$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix')) + + + <_CurrentOS Condition="'$(BuildOs)' != ''">$(BuildOs) + <_CurrentOS Condition="'$(BuildOs)' == '' and $([MSBuild]::IsOSPlatform('Linux'))">all + <_CurrentOS Condition="'$(BuildOs)' == '' and $([MSBuild]::IsOSPlatform('Windows'))">all + <_CurrentOS Condition="'$(BuildOs)' == '' and $([MSBuild]::IsOSPlatform('OSX'))">all + + + + + + + + diff --git a/eng/TestEnumerationRunsheetBuilder/DESIGN.md b/eng/TestEnumerationRunsheetBuilder/DESIGN.md new file mode 100644 index 00000000000..9aae4375461 --- /dev/null +++ b/eng/TestEnumerationRunsheetBuilder/DESIGN.md @@ -0,0 +1,166 @@ +# TestEnumerationRunsheetBuilder Design + +## Overview + +This document describes the design for migrating the current `GetTestProjects.proj` test enumeration mechanism to work through the Arcade SDK's runsheet builder pattern. + +## Current Architecture + +### GetTestProjects.proj (Current) +- **Approach**: Centralized test project discovery and enumeration +- **Invocation**: Manual execution via `dotnet build tests/Shared/GetTestProjects.proj` +- **Process**: + 1. Discovers all test projects using glob patterns + 2. Calls MSBuild on each project to determine GitHub Actions eligibility + 3. Builds split test projects to generate test class lists + 4. Generates final test lists and matrices using PowerShell scripts + +### Problems with Current Approach +- Requires explicit invocation outside the standard build process +- Not integrated with Arcade SDK's runsheet builder mechanism +- Duplicates logic that could be shared with other runsheet builders + +## New Architecture: TestEnumerationRunsheetBuilder + +### Design Principles +1. **Distributed Processing**: Each test project generates its own enumeration data during build +2. **Arcade SDK Integration**: Follows the same pattern as existing runsheet builders +3. **Reuse Existing Logic**: Leverages existing test enumeration and splitting mechanisms +4. **Centralized Combination**: Final aggregation happens in `AfterSolutionBuild.targets` + +### Components + +#### 1. TestEnumerationRunsheetBuilder.targets +- **Location**: `eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets` +- **Purpose**: Runs once per test project to generate test enumeration data +- **Outputs**: Per-project test enumeration files in `ArtifactsTmpDir` + +#### 2. Enhanced AfterSolutionBuild.targets +- **Purpose**: Combines individual test enumeration files into final outputs +- **Trigger**: When `TestRunnerName=TestEnumerationRunsheetBuilder` +- **Outputs**: Same as current GetTestProjects.proj (test lists and matrices) + +### Flow Diagram + +``` +Build Process +├── For each test project: +│ ├── TestEnumerationRunsheetBuilder.targets runs +│ ├── Generates project-specific enumeration data +│ └── Writes to artifacts/tmp/{project}.testenumeration.json +│ +└── After all projects built: + ├── AfterSolutionBuild.targets runs + ├── Collects all testenumeration.json files + ├── Processes split tests (if any) + └── Generates final outputs: + ├── TestsForGithubActions.list + ├── TestsForGithubActions.list.split-projects + └── test-matrices/*.json +``` + +## Implementation Details + +### TestEnumerationRunsheetBuilder.targets + +```msbuild + + + + + + + + + + <_EnumerationData Include="{ + 'project': '$(MSBuildProjectName)', + 'fullPath': '$(MSBuildProjectFullPath)', + 'shortName': '$(_ShortName)', + 'runOnGithubActions': '%(_ProjectInfo.RunTestsOnGithubActions)', + 'splitTests': '%(_ProjectInfo.SplitTests)' + }" /> + + + + + +``` + +### Enhanced AfterSolutionBuild.targets + +```msbuild + + + + + <_TestEnumerationFiles Include="$(ArtifactsTmpDir)/*.testenumeration.json" /> + + + + + <_ProcessingScript> + # PowerShell script to: + # 1. Read all testenumeration.json files + # 2. Filter by OS and eligibility + # 3. Generate test lists and split test lists + # 4. Call existing matrix generation script for split tests + + + + + + +``` + +## Migration Strategy + +### Phase 1: Implementation +1. Create `TestEnumerationRunsheetBuilder.targets` +2. Enhance `AfterSolutionBuild.targets` with test enumeration logic +3. Implement PowerShell processing script + +### Phase 2: Integration +1. Update GitHub Actions workflows to use new approach +2. Test compatibility with existing split test functionality +3. Validate output format matches current GetTestProjects.proj + +### Phase 3: Cleanup +1. Deprecate GetTestProjects.proj usage in workflows +2. Remove manual invocation commands +3. Document new usage pattern + +## Usage + +### Command Line +```bash +# Instead of manual GetTestProjects.proj invocation: +dotnet build tests/Shared/GetTestProjects.proj /bl:artifacts/log/Debug/GetTestProjects.binlog /p:TestsListOutputPath=artifacts/TestsForGithubActions.list /p:TestMatrixOutputPath=artifacts/test-matrices/ /p:ContinuousIntegrationBuild=true /p:BuildOs=linux + +# New approach using runsheet builder: +./build.cmd -test /p:TestRunnerName=TestEnumerationRunsheetBuilder /p:TestsListOutputPath=artifacts/TestsForGithubActions.list /p:TestMatrixOutputPath=artifacts/test-matrices/ /p:ContinuousIntegrationBuild=true /p:BuildOs=linux +``` + +### Integration with CI +The new approach integrates seamlessly with the existing build infrastructure and requires minimal changes to GitHub Actions workflows. + +## Benefits + +1. **Consistency**: Follows the same pattern as other runsheet builders +2. **Automatic Discovery**: No manual project enumeration required +3. **Build Integration**: Leverages existing build process and caching +4. **Maintainability**: Reduces code duplication and improves consistency +5. **Extensibility**: Easy to add new test enumeration features + +## Backward Compatibility + +- Existing GetTestProjects.proj functionality remains unchanged +- New approach generates identical output formats +- Migration can be done incrementally per workflow \ No newline at end of file diff --git a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets new file mode 100644 index 00000000000..caa87ccad9f --- /dev/null +++ b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets @@ -0,0 +1,133 @@ + + + + + + <_ShouldSkipProject>false + <_ShouldSkipProject Condition="$(MSBuildProjectDirectory.Contains('tests\Shared'))">true + <_ShouldSkipProject Condition="$(MSBuildProjectDirectory.Contains('tests\testproject'))">true + <_ShouldSkipProject Condition="$(MSBuildProjectDirectory.Contains('tests\TestingAppHost1'))">true + + + <_ShortName>$([System.IO.Path]::GetFileNameWithoutExtension('$(MSBuildProjectName)').Replace('Aspire.', '').Replace('.Tests', '')) + + + + + + + <_CurrentProject Include="$(MSBuildProjectFullPath)" /> + + + + + + + + + + + + + + + + <_RelativeProjectPath>$([System.String]::Copy('$(MSBuildProjectFullPath)').Replace('$(RepoRoot)', '')) + + <_RelativeProjectPath Condition="$(_RelativeProjectPath.StartsWith('/')) or $(_RelativeProjectPath.StartsWith('\'))">$(_RelativeProjectPath.Substring(1)) + + + + <_HelixDir Condition="'$(TestArchiveTestsDir)' != ''">$(TestArchiveTestsDir) + <_HelixDir Condition="'$(TestArchiveTestsDir)' == ''">$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix')) + <_TestListFile>$(_HelixDir)$(MSBuildProjectName).tests.list + <_MetadataFile>$(_HelixDir)$(MSBuildProjectName).tests.metadata.json + <_HasTestMetadata Condition="Exists('$(_MetadataFile)')">true + <_HasTestMetadata Condition="'$(_HasTestMetadata)' != 'true'">false + + + <_OsArrayJson>%(_ProjectInfo.SupportedOSes) + + + <_EnumerationJson>{ + "project": "$(MSBuildProjectName)", + "fullPath": "$(_RelativeProjectPath)", + "shortName": "$(_ShortName)", + "supportedOSes": [$(_OsArrayJson)], + "splitTests": "%(_ProjectInfo.SplitTests)", + "hasTestMetadata": "$(_HasTestMetadata)", + "testListFile": "$(_TestListFile.Replace('$(RepoRoot)', ''))", + "metadataFile": "$(_MetadataFile.Replace('$(RepoRoot)', ''))" +} + + + + + + + + + + + + diff --git a/eng/Testing.props b/eng/Testing.props index d24ccae9410..5fad18f14ca 100644 --- a/eng/Testing.props +++ b/eng/Testing.props @@ -2,6 +2,12 @@ true + + 20m + 10m + 15m + 10m + true true diff --git a/eng/Testing.targets b/eng/Testing.targets index 853a3949d3b..e31f6bf0186 100644 --- a/eng/Testing.targets +++ b/eng/Testing.targets @@ -109,4 +109,143 @@ + + + + <_ShouldSplit>false + <_ShouldSplit Condition="'$(SplitTestsOnCI)' == 'true'">true + + + + + <_ProjectInfo Condition="'$(BuildOs)' == 'windows'" + Include="$(MSBuildProjectFullPath)" + RunTestsOnGithubActions="$(RunOnGithubActionsWindows)" + SplitTests="$(_ShouldSplit)" + MSBuildSourceProjectFile="$(MSBuildProjectFullPath)" /> + <_ProjectInfo Condition="'$(BuildOs)' == 'linux'" + Include="$(MSBuildProjectFullPath)" + RunTestsOnGithubActions="$(RunOnGithubActionsLinux)" + SplitTests="$(_ShouldSplit)" + MSBuildSourceProjectFile="$(MSBuildProjectFullPath)" /> + <_ProjectInfo Condition="'$(BuildOs)' == 'darwin'" + Include="$(MSBuildProjectFullPath)" + RunTestsOnGithubActions="$(RunOnGithubActionsMacOS)" + SplitTests="$(_ShouldSplit)" + MSBuildSourceProjectFile="$(MSBuildProjectFullPath)" /> + + + <_ProjectInfo Condition="'$(BuildOs)' == ''" + Include="$(MSBuildProjectFullPath)" + RunTestsOnGithubActions="$(RunOnGithubActions)" + SplitTests="$(_ShouldSplit)" + MSBuildSourceProjectFile="$(MSBuildProjectFullPath)" /> + + + + + + + <_ShouldSplit>false + <_ShouldSplit Condition="'$(SplitTestsOnCI)' == 'true'">true + + + <_SupportedOSes> + <_SupportedOSes Condition="'$(RunOnGithubActionsWindows)' == 'true'">$(_SupportedOSes)"windows", + <_SupportedOSes Condition="'$(RunOnGithubActionsLinux)' == 'true'">$(_SupportedOSes)"linux", + <_SupportedOSes Condition="'$(RunOnGithubActionsMacOS)' == 'true'">$(_SupportedOSes)"macos", + + <_SupportedOSes Condition="'$(_SupportedOSes)' != ''">$(_SupportedOSes.TrimEnd(',')) + + + + <_ProjectInfo Include="$(MSBuildProjectFullPath)" + SupportedOSes="$(_SupportedOSes)" + SplitTests="$(_ShouldSplit)" + MSBuildSourceProjectFile="$(MSBuildProjectFullPath)" /> + + + + + + + + <_HelixDir Condition="'$(TestArchiveTestsDir)' != ''">$(TestArchiveTestsDir) + <_HelixDir Condition="'$(TestArchiveTestsDir)' == ''">$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix')) + <_MetadataFile>$(_HelixDir)$(MSBuildProjectName).tests.metadata.json + + + <_RelativeProjectPath>$([System.String]::Copy('$(MSBuildProjectFullPath)').Replace('$(RepoRoot)', '')) + + <_RelativeProjectPath Condition="$(_RelativeProjectPath.StartsWith('/')) or $(_RelativeProjectPath.StartsWith('\\'))">$(_RelativeProjectPath.Substring(1)) + + + <_RequiresNugets>false + <_RequiresNugets Condition="'$(RequiresNugetsForSplitTests)' == 'true'">true + + + <_RequiresTestSdk>false + <_RequiresTestSdk Condition="'$(RequiresTestSdkForSplitTests)' == 'true'">true + + + + <_EnablePlaywrightInstall>false + <_EnablePlaywrightInstall Condition="'$(EnablePlaywrightInstallForSplitTests)' == 'true'">true + + + <_SupportedOSesJson> + <_SupportedOSesJson Condition="'$(RunOnGithubActionsWindows)' == 'true'">$(_SupportedOSesJson)"windows", + <_SupportedOSesJson Condition="'$(RunOnGithubActionsLinux)' == 'true'">$(_SupportedOSesJson)"linux", + <_SupportedOSesJson Condition="'$(RunOnGithubActionsMacOS)' == 'true'">$(_SupportedOSesJson)"macos", + + <_SupportedOSesJson Condition="'$(_SupportedOSesJson)' != ''">$(_SupportedOSesJson.TrimEnd(',')) + + + <_TestSessionTimeout Condition="'$(TestSessionTimeout)' != ''">$(TestSessionTimeout) + <_TestSessionTimeout Condition="'$(TestSessionTimeout)' == ''">20m + <_TestHangTimeout Condition="'$(TestHangTimeout)' != ''">$(TestHangTimeout) + <_TestHangTimeout Condition="'$(TestHangTimeout)' == ''">10m + <_UncollectedTestsSessionTimeout Condition="'$(UncollectedTestsSessionTimeout)' != ''">$(UncollectedTestsSessionTimeout) + <_UncollectedTestsSessionTimeout Condition="'$(UncollectedTestsSessionTimeout)' == ''">15m + <_UncollectedTestsHangTimeout Condition="'$(UncollectedTestsHangTimeout)' != ''">$(UncollectedTestsHangTimeout) + <_UncollectedTestsHangTimeout Condition="'$(UncollectedTestsHangTimeout)' == ''">10m + + + + <_ShortName>$([System.IO.Path]::GetFileNameWithoutExtension('$(MSBuildProjectName)').Replace('Aspire.', '').Replace('.Tests', '')) + + + <_MetadataJson>{ + "projectName": "$(MSBuildProjectName)", + "shortName": "$(_ShortName)", + "testClassNamesPrefix": "$(MSBuildProjectName)", + "testProjectPath": "$(_RelativeProjectPath)", + "requiresNugets": "$(_RequiresNugets.ToLowerInvariant())", + "requiresTestSdk": "$(_RequiresTestSdk.ToLowerInvariant())", + "enablePlaywrightInstall": "$(_EnablePlaywrightInstall.ToLowerInvariant())", + "supportedOSes": [$(_SupportedOSesJson)], + "testSessionTimeout": "$(_TestSessionTimeout)", + "testHangTimeout": "$(_TestHangTimeout)", + "uncollectedTestsSessionTimeout": "15m", + "uncollectedTestsHangTimeout": "10m" +} + + + + + + + + + diff --git a/eng/scripts/build-test-matrix.ps1 b/eng/scripts/build-test-matrix.ps1 new file mode 100644 index 00000000000..9cef64e3179 --- /dev/null +++ b/eng/scripts/build-test-matrix.ps1 @@ -0,0 +1,366 @@ +<# +.SYNOPSIS + Builds the combined test matrix for GitHub Actions from test enumeration files. + +.DESCRIPTION + This script consolidates the functionality of process-test-enumeration.ps1 and + generate-test-matrix.ps1 into a single optimized script that: + 1. Collects all .testenumeration.json files + 2. Filters tests by supported OSes + 3. Separates regular tests from split tests + 4. Generates matrix entries for all tests (with partition/class splitting) + 5. Writes the final combined-tests-matrix.json in a single pass + + No intermediate files are created - all data processing happens in memory. + +.PARAMETER ArtifactsTmpDir + Directory containing .testenumeration.json files from test projects. + +.PARAMETER ArtifactsHelixDir + Directory containing .tests.list and .tests.metadata.json files. + +.PARAMETER OutputMatrixFile + Path to write the combined test matrix JSON file. + +.PARAMETER TestsListOutputFile + Optional path to write backward-compatible test list file (regular tests only) used on AzDO + +.PARAMETER CurrentOS + Current operating system (linux, windows, macos). Filters tests by supported OSes. + If not specified or set to 'all', includes tests for all OSes without filtering. + +.NOTES + PowerShell 7+ + Replaces: process-test-enumeration.ps1 + generate-test-matrix.ps1 +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory=$true)] + [string]$ArtifactsTmpDir, + + [Parameter(Mandatory=$true)] + [string]$ArtifactsHelixDir, + + [Parameter(Mandatory=$true)] + [string]$OutputMatrixFile, + + [Parameter(Mandatory=$false)] + [string]$TestsListOutputFile = "", + + [Parameter(Mandatory=$false)] + [string]$CurrentOS = "all" +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +# Normalize OS name +$CurrentOS = $CurrentOS.ToLowerInvariant() + +# Determine if we should filter by OS +$filterByOS = $CurrentOS -ne 'all' + +if ($filterByOS) { + Write-Host "Building test matrix for OS: $CurrentOS" +} else { + Write-Host "Building combined test matrix for all OSes" +} +Write-Host "Enumerations directory: $ArtifactsTmpDir" +Write-Host "Helix directory: $ArtifactsHelixDir" + +# Helper function to create matrix entry for regular (non-split) tests +function New-RegularTestEntry { + param( + [Parameter(Mandatory=$true)] + $Enumeration, + [Parameter(Mandatory=$false)] + $Metadata = $null + ) + + $entry = [ordered]@{ + type = 'regular' + projectName = $Enumeration.project + name = $Enumeration.shortName + shortname = $Enumeration.shortName + testProjectPath = $Enumeration.fullPath + workitemprefix = $Enumeration.project + } + + # Add metadata if available + if ($Metadata) { + if ($Metadata.PSObject.Properties['testSessionTimeout']) { $entry['testSessionTimeout'] = $Metadata.testSessionTimeout } + if ($Metadata.PSObject.Properties['testHangTimeout']) { $entry['testHangTimeout'] = $Metadata.testHangTimeout } + if ($Metadata.PSObject.Properties['requiresNugets'] -and $Metadata.requiresNugets -eq 'true') { $entry['requiresNugets'] = 'true' } + if ($Metadata.PSObject.Properties['requiresTestSdk'] -and $Metadata.requiresTestSdk -eq 'true') { $entry['requiresTestSdk'] = 'true' } + if ($Metadata.PSObject.Properties['extraTestArgs'] -and $Metadata.extraTestArgs) { $entry['extraTestArgs'] = $Metadata.extraTestArgs } + } + + # Add supported OSes + $entry['supportedOSes'] = @($Enumeration.supportedOSes) + + return $entry +} + +# Helper function to create matrix entry for collection-based split tests +function New-CollectionTestEntry { + param( + [Parameter(Mandatory=$true)] + [string]$CollectionName, + [Parameter(Mandatory=$true)] + $Metadata, + [Parameter(Mandatory=$true)] + [bool]$IsUncollected + ) + + $suffix = if ($IsUncollected) { 'uncollected' } else { $CollectionName } + $baseShortName = if ($Metadata.shortName) { $Metadata.shortName } else { $Metadata.projectName } + + $entry = [ordered]@{ + type = 'collection' + projectName = $Metadata.projectName + name = if ($IsUncollected) { $baseShortName } else { "$baseShortName-$suffix" } + shortname = if ($IsUncollected) { $baseShortName } else { "$baseShortName-$suffix" } + testProjectPath = $Metadata.testProjectPath + workitemprefix = "$($Metadata.projectName)_$suffix" + collection = $CollectionName + } + + # Use uncollected timeouts if available, otherwise use regular + if ($IsUncollected) { + if ($Metadata.PSObject.Properties['uncollectedTestsSessionTimeout']) { + $entry['testSessionTimeout'] = $Metadata.uncollectedTestsSessionTimeout + } elseif ($Metadata.PSObject.Properties['testSessionTimeout']) { + $entry['testSessionTimeout'] = $Metadata.testSessionTimeout + } + + if ($Metadata.PSObject.Properties['uncollectedTestsHangTimeout']) { + $entry['testHangTimeout'] = $Metadata.uncollectedTestsHangTimeout + } elseif ($Metadata.PSObject.Properties['testHangTimeout']) { + $entry['testHangTimeout'] = $Metadata.testHangTimeout + } + } else { + if ($Metadata.PSObject.Properties['testSessionTimeout']) { $entry['testSessionTimeout'] = $Metadata.testSessionTimeout } + if ($Metadata.PSObject.Properties['testHangTimeout']) { $entry['testHangTimeout'] = $Metadata.testHangTimeout } + } + + if ($Metadata.PSObject.Properties['requiresNugets'] -and $Metadata.requiresNugets -eq 'true') { $entry['requiresNugets'] = 'true' } + if ($Metadata.PSObject.Properties['requiresTestSdk'] -and $Metadata.requiresTestSdk -eq 'true') { $entry['requiresTestSdk'] = 'true' } + + # Add test filter for collection-based splitting + if ($IsUncollected) { + $entry['extraTestArgs'] = '--filter-not-trait "Partition=*"' + } else { + $entry['extraTestArgs'] = "--filter-trait `"Partition=$CollectionName`"" + } + + # Add supported OSes from metadata (should match enumeration) + if ($Metadata.PSObject.Properties['supportedOSes']) { + $entry['supportedOSes'] = @($Metadata.supportedOSes) + } + + return $entry +} + +# Helper function to create matrix entry for class-based split tests +function New-ClassTestEntry { + param( + [Parameter(Mandatory=$true)] + [string]$ClassName, + [Parameter(Mandatory=$true)] + $Metadata + ) + + # Extract short class name (last segment after last dot) + $shortClassName = $ClassName.Split('.')[-1] + $baseShortName = if ($Metadata.shortName) { $Metadata.shortName } else { $Metadata.projectName } + + $entry = [ordered]@{ + type = 'class' + projectName = $Metadata.projectName + name = "$baseShortName-$shortClassName" + shortname = "$baseShortName-$shortClassName" + testProjectPath = $Metadata.testProjectPath + workitemprefix = "$($Metadata.projectName)_$shortClassName" + classname = $ClassName + } + + if ($Metadata.PSObject.Properties['testSessionTimeout']) { $entry['testSessionTimeout'] = $Metadata.testSessionTimeout } + if ($Metadata.PSObject.Properties['testHangTimeout']) { $entry['testHangTimeout'] = $Metadata.testHangTimeout } + if ($Metadata.PSObject.Properties['requiresNugets'] -and $Metadata.requiresNugets -eq 'true') { $entry['requiresNugets'] = 'true' } + if ($Metadata.PSObject.Properties['requiresTestSdk'] -and $Metadata.requiresTestSdk -eq 'true') { $entry['requiresTestSdk'] = 'true' } + + # Add test filter for class-based splitting + $entry['extraTestArgs'] = "--filter-class `"$ClassName`"" + + # Add supported OSes from metadata + if ($Metadata.PSObject.Properties['supportedOSes']) { + $entry['supportedOSes'] = @($Metadata.supportedOSes) + } + + return $entry +} + +# 1. Collect all enumeration files +$enumerationFiles = @(Get-ChildItem -Path $ArtifactsTmpDir -Filter "*.testenumeration.json" -ErrorAction SilentlyContinue) + +if ($enumerationFiles.Count -eq 0) { + Write-Warning "No test enumeration files found in $ArtifactsTmpDir" + # Create empty matrix + $matrix = @{ include = @() } + $matrix | ConvertTo-Json -Depth 10 -Compress | Set-Content -Path $OutputMatrixFile + Write-Host "Created empty test matrix: $OutputMatrixFile" + exit 0 +} + +Write-Host "Found $($enumerationFiles.Count) test enumeration file(s)" + +# 2. Build matrix entries +$matrixEntries = [System.Collections.Generic.List[object]]::new() +$regularTestsList = [System.Collections.Generic.List[string]]::new() + +foreach ($enumFile in $enumerationFiles) { + $enum = Get-Content -Raw -Path $enumFile.FullName | ConvertFrom-Json + + Write-Host "Processing: $($enum.project)" + + # Filter by supported OSes (skip if current OS not supported) + # Only filter if a specific OS was requested + if ($filterByOS -and $enum.supportedOSes -and $enum.supportedOSes.Count -gt 0) { + $osSupported = $false + foreach ($os in $enum.supportedOSes) { + if ($os.ToLowerInvariant() -eq $CurrentOS) { + $osSupported = $true + break + } + } + + if (-not $osSupported) { + Write-Host " ⊘ Skipping (not supported on $CurrentOS)" + continue + } + } + + # Check if this is a split test with metadata + if ($enum.splitTests -eq 'true' -and $enum.hasTestMetadata -eq 'true') { + Write-Host " → Split test (processing partitions/classes)" + + # Read metadata and test list - use paths from enumeration file if available + if ($enum.metadataFile) { + # Path is relative to repo root, make it absolute + $metaFile = Join-Path $PSScriptRoot "../../$($enum.metadataFile)" -Resolve -ErrorAction SilentlyContinue + if (-not $metaFile) { + $metaFile = [System.IO.Path]::Combine($PSScriptRoot, "../..", $enum.metadataFile) + } + } else { + $metaFile = Join-Path $ArtifactsHelixDir "$($enum.project).tests.metadata.json" + } + + if ($enum.testListFile) { + # Path is relative to repo root, make it absolute + $listFile = Join-Path $PSScriptRoot "../../$($enum.testListFile)" -Resolve -ErrorAction SilentlyContinue + if (-not $listFile) { + $listFile = [System.IO.Path]::Combine($PSScriptRoot, "../..", $enum.testListFile) + } + } else { + $listFile = Join-Path $ArtifactsHelixDir "$($enum.project).tests.list" + } + + if (-not (Test-Path $metaFile)) { + Write-Warning " ⚠ Metadata file not found: $metaFile" + continue + } + + if (-not (Test-Path $listFile)) { + Write-Warning " ⚠ Test list file not found: $listFile" + continue + } + + $metadata = Get-Content -Raw -Path $metaFile | ConvertFrom-Json + + # Add supported OSes to metadata from enumeration + $metadata | Add-Member -Force -MemberType NoteProperty -Name 'supportedOSes' -Value $enum.supportedOSes + + $listLines = Get-Content -Path $listFile + + $partitionCount = 0 + $classCount = 0 + + foreach ($line in $listLines) { + $line = $line.Trim() + if ([string]::IsNullOrWhiteSpace($line)) { continue } + + if ($line -match '^collection:(.+)$') { + # Collection/partition entry + $collectionName = $Matches[1] + $entry = New-CollectionTestEntry -CollectionName $collectionName -Metadata $metadata -IsUncollected $false + $matrixEntries.Add($entry) + $partitionCount++ + } + elseif ($line -match '^uncollected:\*$') { + # Uncollected tests entry + $entry = New-CollectionTestEntry -CollectionName '*' -Metadata $metadata -IsUncollected $true + $matrixEntries.Add($entry) + $partitionCount++ + } + elseif ($line -match '^class:(.+)$') { + # Class-based entry + $className = $Matches[1] + $entry = New-ClassTestEntry -ClassName $className -Metadata $metadata + $matrixEntries.Add($entry) + $classCount++ + } + } + + Write-Host " ✓ Added $partitionCount partition(s) and $classCount class(es)" + } + else { + # Regular (non-split) test + #Write-Host " → Regular test" + + # Try to load metadata if available - use path from enumeration file if available + if ($enum.metadataFile) { + # Path is relative to repo root, make it absolute + $metaFile = Join-Path $PSScriptRoot "../../$($enum.metadataFile)" -Resolve -ErrorAction SilentlyContinue + if (-not $metaFile) { + $metaFile = [System.IO.Path]::Combine($PSScriptRoot, "../..", $enum.metadataFile) + } + } else { + $metaFile = Join-Path $ArtifactsHelixDir "$($enum.project).tests.metadata.json" + } + + $metadata = $null + if (Test-Path $metaFile) { + $metadata = Get-Content -Raw -Path $metaFile | ConvertFrom-Json + } + + $entry = New-RegularTestEntry -Enumeration $enum -Metadata $metadata + $matrixEntries.Add($entry) + $regularTestsList.Add($enum.shortName) + + #Write-Host " ✓ Added regular test" + } +} + +# 3. Write final matrix +Write-Host "" +Write-Host "Generated $($matrixEntries.Count) total matrix entries" + +$matrix = @{ include = $matrixEntries } +$outputDir = [System.IO.Path]::GetDirectoryName($OutputMatrixFile) +if ($outputDir -and -not (Test-Path $outputDir)) { + New-Item -ItemType Directory -Path $outputDir -Force | Out-Null +} + +$matrix | ConvertTo-Json -Depth 10 -Compress | Set-Content -Path $OutputMatrixFile -Encoding UTF8 + +Write-Host "✓ Matrix written to: $OutputMatrixFile" + +# 4. Write backward-compatible test list if requested +if ($TestsListOutputFile) { + $regularTestsList | Set-Content -Path $TestsListOutputFile -Encoding UTF8 + Write-Host "✓ Test list written to: $TestsListOutputFile" +} + +Write-Host "" +Write-Host "Matrix build complete!" diff --git a/eng/scripts/split-test-projects-for-ci.ps1 b/eng/scripts/split-test-projects-for-ci.ps1 new file mode 100644 index 00000000000..3b55314d01d --- /dev/null +++ b/eng/scripts/split-test-projects-for-ci.ps1 @@ -0,0 +1,213 @@ +<# +.SYNOPSIS + Extract test metadata (collections or classes) from test assemblies. + +.DESCRIPTION + Determines splitting mode by extracting Collection and Trait attributes from the test assembly: + - Uses ExtractTestPartitions tool to find [Collection("name")] or [Trait("Partition", "name")] attributes + - If partitions found → partition mode (collections) + - Else → class mode (runs --list-tests to enumerate classes) + Outputs a .tests.list file with either: + collection:Name + ... + uncollected:* (always appended in collection mode) + OR + class:Full.Namespace.ClassName + ... + + Also updates the per-project metadata JSON with mode and collections. + +.PARAMETER TestAssemblyPath + Path to the test assembly DLL for extracting partition attributes. + +.PARAMETER RunCommand + The command to run the test assembly (e.g., "dotnet exec "). + Only invoked if partition extraction fails and class-based splitting is needed. + +.PARAMETER TestClassNamesPrefix + Namespace prefix used to recognize test classes (e.g. Aspire.Templates.Tests). + +.PARAMETER TestCollectionsToSkip + Semicolon-separated collection names to exclude from dedicated jobs. + +.PARAMETER OutputListFile + Path to the .tests.list output file. + +.PARAMETER MetadataJsonFile + Path to the .tests.metadata.json file (script may append mode info). + +.PARAMETER RepoRoot + Path to the repository root (for locating the ExtractTestPartitions tool). + +.NOTES + PowerShell 7+ + Fails fast if zero test classes discovered when in class mode. + Optimized to only run --list-tests when partition extraction fails. +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory=$true)] + [string]$TestAssemblyPath, + + [Parameter(Mandatory=$true)] + [string]$RunCommand, + + [Parameter(Mandatory=$true)] + [string]$TestClassNamesPrefix, + + [Parameter(Mandatory=$false)] + [string]$TestCollectionsToSkip = "", + + [Parameter(Mandatory=$true)] + [string]$OutputListFile, + + [Parameter(Mandatory=$false)] + [string]$MetadataJsonFile = "", + + [Parameter(Mandatory=$true)] + [string]$RepoRoot +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +if (-not (Test-Path $TestAssemblyPath)) { + Write-Error "TestAssemblyPath not found: $TestAssemblyPath" +} + +$collections = [System.Collections.Generic.HashSet[string]]::new() +$classes = [System.Collections.Generic.HashSet[string]]::new() + +# Extract partitions using the ExtractTestPartitions tool +# This step is optional - if it fails, we'll fall back to class-based splitting +$partitionsFile = Join-Path ([System.IO.Path]::GetTempPath()) "partitions-$([System.Guid]::NewGuid()).txt" +try { + $toolPath = Join-Path $RepoRoot "artifacts/bin/ExtractTestPartitions/Debug/net8.0/ExtractTestPartitions.dll" + + # Build the tool if it doesn't exist + if (-not (Test-Path $toolPath)) { + Write-Host "Building ExtractTestPartitions tool..." + $toolProjectPath = Join-Path $RepoRoot "tools/ExtractTestPartitions/ExtractTestPartitions.csproj" + & dotnet build $toolProjectPath -c Debug --nologo -v quiet + if ($LASTEXITCODE -ne 0) { + Write-Host "Warning: Failed to build ExtractTestPartitions tool. Using class-based splitting." + } + } + + if (Test-Path $toolPath) { + Write-Host "Extracting partitions from assembly: $TestAssemblyPath" + $toolOutput = & dotnet $toolPath --assembly-path $TestAssemblyPath --output-file $partitionsFile 2>&1 + $toolExitCode = $LASTEXITCODE + + # Display tool output (informational) + if ($toolOutput) { + $toolOutput | Write-Host + } + + # If partitions file was created, read it (even if exit code is non-zero) + if (Test-Path $partitionsFile) { + $partitionLines = Get-Content $partitionsFile -ErrorAction SilentlyContinue + if ($partitionLines) { + foreach ($partition in $partitionLines) { + if (-not [string]::IsNullOrWhiteSpace($partition)) { + $collections.Add($partition.Trim()) | Out-Null + } + } + Write-Host "Found $($collections.Count) partition(s) via attribute extraction" + } + } + elseif ($toolExitCode -ne 0) { + Write-Host "Partition extraction completed with warnings. Falling back to class-based splitting." + } + } +} catch { + # Partition extraction is optional - if it fails, we fall back to class-based splitting + Write-Host "Partition extraction encountered an issue. Falling back to class-based splitting." + Write-Host "Details: $_" +} +finally { + # Clean up temp file + if (Test-Path $partitionsFile) { + Remove-Item $partitionsFile -ErrorAction SilentlyContinue + } +} + +# Apply collection filtering +$skipList = @() +if ($TestCollectionsToSkip) { + $skipList = $TestCollectionsToSkip -split ';' | ForEach-Object { $_.Trim() } | Where-Object { $_ } +} + +$filteredCollections = @($collections | Where-Object { $skipList -notcontains $_ }) + +# Determine mode: if we have partitions, use collection mode; otherwise fall back to class mode +$mode = if ($filteredCollections.Count -gt 0) { 'collection' } else { 'class' } + +# Only run --list-tests if we need class-based splitting (no partitions found) +if ($mode -eq 'class') { + Write-Host "No partitions found. Running --list-tests to extract class names..." + + # Run the test assembly with --list-tests to get all test names + $testOutput = & $RunCommand --filter-not-trait category=failing --list-tests 2>&1 + + if ($LASTEXITCODE -ne 0) { + Write-Warning "Test listing command failed with exit code $LASTEXITCODE. Attempting to parse partial output..." + } + + # Extract class names from test listing + $classNamePattern = '^(\s*)' + [Regex]::Escape($TestClassNamesPrefix) + '\.([^\.]+)\.' + + foreach ($line in $testOutput) { + $lineStr = $line.ToString() + # Extract class name from test name + # Format: " Namespace.ClassName.MethodName(...)" or "Namespace.ClassName.MethodName" + if ($lineStr -match $classNamePattern) { + $className = "$TestClassNamesPrefix.$($Matches[2])" + $classes.Add($className) | Out-Null + } + } + + if ($classes.Count -eq 0) { + Write-Error "No test classes discovered matching prefix '$TestClassNamesPrefix'." + } +} + +$outputDir = [System.IO.Path]::GetDirectoryName($OutputListFile) +if ($outputDir -and -not (Test-Path $outputDir)) { + New-Item -ItemType Directory -Path $outputDir -Force | Out-Null +} + +$lines = [System.Collections.Generic.List[string]]::new() + +if ($mode -eq 'collection') { + foreach ($c in ($filteredCollections | Sort-Object)) { + $lines.Add("collection:$c") + } + $lines.Add("uncollected:*") +} else { + foreach ($cls in ($classes | Sort-Object)) { + $lines.Add("class:$cls") + } +} + +$lines | Set-Content -Path $OutputListFile -Encoding UTF8 + +if ($MetadataJsonFile -and (Test-Path $MetadataJsonFile)) { + try { + $meta = Get-Content -Raw -Path $MetadataJsonFile | ConvertFrom-Json + # Add or update properties + $meta | Add-Member -Force -MemberType NoteProperty -Name 'mode' -Value $mode + $meta | Add-Member -Force -MemberType NoteProperty -Name 'collections' -Value @($filteredCollections | Sort-Object) + $meta | Add-Member -Force -MemberType NoteProperty -Name 'classCount' -Value $classes.Count + $meta | Add-Member -Force -MemberType NoteProperty -Name 'collectionCount' -Value $filteredCollections.Count + $meta | ConvertTo-Json -Depth 20 | Set-Content -Path $MetadataJsonFile -Encoding UTF8 + } catch { + Write-Warning "Failed updating metadata JSON: $_" + } +} + +Write-Host "Mode: $mode" +Write-Host "Collections discovered (after filtering): $($filteredCollections.Count)" +Write-Host "Classes discovered: $($classes.Count)" +Write-Host "Output list written: $OutputListFile" diff --git a/tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj b/tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj index ae826d9bb89..3ac5f69bbd1 100644 --- a/tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj +++ b/tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj @@ -17,6 +17,8 @@ false testassets\testproject\ + true + <_BuildForTestsRunningOutsideOfRepo Condition="'$(TestsRunningOutsideOfRepo)' == 'true' or '$(ContinuousIntegrationBuild)' == 'true'">true $(_BuildForTestsRunningOutsideOfRepo) BUILD_FOR_TESTS_RUNNING_OUTSIDE_OF_REPO;$(DefineConstants) diff --git a/tests/Aspire.Hosting.Python.Tests/AddPythonAppTests.cs b/tests/Aspire.Hosting.Python.Tests/AddPythonAppTests.cs index fbf4d2e3a44..ead38eb032f 100644 --- a/tests/Aspire.Hosting.Python.Tests/AddPythonAppTests.cs +++ b/tests/Aspire.Hosting.Python.Tests/AddPythonAppTests.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. #pragma warning disable CS0612 @@ -476,6 +476,7 @@ public void WithVirtualEnvironment_ThrowsOnNullOrEmptyPath() } [Fact] + [QuarantinedTest("https://foo.com/issue/123")] public async Task WithVirtualEnvironment_CanBeChainedWithOtherExtensions() { using var builder = TestDistributedApplicationBuilder.Create().WithTestAndResourceLogging(outputHelper); diff --git a/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj b/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj index f55080a25d1..f5c00c87ed7 100644 --- a/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj +++ b/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj @@ -10,6 +10,16 @@ false false + + + true + Aspire.Hosting.Tests + + + 30m + 15m + 20m + 10m diff --git a/tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs b/tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs index b8b75ce0b91..da8b961798b 100644 --- a/tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs +++ b/tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs @@ -29,6 +29,8 @@ namespace Aspire.Hosting.Tests; +[Collection("DistributedApplicationTests")] +[Trait("Partition", "DistributedApplicationTests")] public class DistributedApplicationTests { private readonly ITestOutputHelper _testOutputHelper; diff --git a/tests/Aspire.Hosting.Tests/MSBuildTests.cs b/tests/Aspire.Hosting.Tests/MSBuildTests.cs index c1501234b68..d2072cb5506 100644 --- a/tests/Aspire.Hosting.Tests/MSBuildTests.cs +++ b/tests/Aspire.Hosting.Tests/MSBuildTests.cs @@ -6,6 +6,7 @@ namespace Aspire.Hosting.Tests; +[Trait("Partition", "Partition1")] public class MSBuildTests { /// diff --git a/tests/Aspire.Hosting.Tests/Publishing/ResourceContainerImageBuilderTests.cs b/tests/Aspire.Hosting.Tests/Publishing/ResourceContainerImageBuilderTests.cs index 7af0a8d3306..48e9a7c722f 100644 --- a/tests/Aspire.Hosting.Tests/Publishing/ResourceContainerImageBuilderTests.cs +++ b/tests/Aspire.Hosting.Tests/Publishing/ResourceContainerImageBuilderTests.cs @@ -13,6 +13,7 @@ namespace Aspire.Hosting.Tests.Publishing; +[Trait("Partition", "Partition1")] public class ResourceContainerImageBuilderTests(ITestOutputHelper output) { [Fact] @@ -803,15 +804,15 @@ public async Task ResolveValue_FormatsDecimalWithInvariantCulture() // Test decimal value var result = await ResourceContainerImageBuilder.ResolveValue(3.14, CancellationToken.None); Assert.Equal("3.14", result); - + // Test double value result = await ResourceContainerImageBuilder.ResolveValue(3.14d, CancellationToken.None); Assert.Equal("3.14", result); - + // Test float value result = await ResourceContainerImageBuilder.ResolveValue(3.14f, CancellationToken.None); Assert.Equal("3.14", result); - + // Test integer (should also work) result = await ResourceContainerImageBuilder.ResolveValue(42, CancellationToken.None); Assert.Equal("42", result); diff --git a/tests/Aspire.Hosting.Tests/WaitForTests.cs b/tests/Aspire.Hosting.Tests/WaitForTests.cs index 559d8f703d8..e6639bbafab 100644 --- a/tests/Aspire.Hosting.Tests/WaitForTests.cs +++ b/tests/Aspire.Hosting.Tests/WaitForTests.cs @@ -11,6 +11,8 @@ namespace Aspire.Hosting.Tests; +[Collection("WaitForTests")] +[Trait("Partition", "WaitForTests")] public class WaitForTests(ITestOutputHelper testOutputHelper) { [Fact] diff --git a/tests/Aspire.Hosting.Tests/WithUrlsTests.cs b/tests/Aspire.Hosting.Tests/WithUrlsTests.cs index ce1a3ce9842..29fb142e585 100644 --- a/tests/Aspire.Hosting.Tests/WithUrlsTests.cs +++ b/tests/Aspire.Hosting.Tests/WithUrlsTests.cs @@ -11,6 +11,7 @@ namespace Aspire.Hosting.Tests; +[Trait("Partition", "Partition1")] public class WithUrlsTests { [Fact] diff --git a/tests/Aspire.Playground.Tests/Aspire.Playground.Tests.csproj b/tests/Aspire.Playground.Tests/Aspire.Playground.Tests.csproj index 6b668f0186a..b52441b8353 100644 --- a/tests/Aspire.Playground.Tests/Aspire.Playground.Tests.csproj +++ b/tests/Aspire.Playground.Tests/Aspire.Playground.Tests.csproj @@ -1,4 +1,4 @@ - + $(DefaultTargetFramework) @@ -105,6 +105,7 @@ + diff --git a/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj b/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj index cfd44eedfd5..31aca3598d5 100644 --- a/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj +++ b/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj @@ -13,8 +13,18 @@ true true - true - Aspire.Templates.Tests + true + Aspire.Templates.Tests + + + + true + true + true + + + 20m + 15m - + - + - - - + - <_Regex>^\s*($(ExtractTestClassNamesPrefix)[^\($]+) + + <_DiscoveryScriptPath>$(RepoRoot)eng\scripts\split-test-projects-for-ci.ps1 + + + <_TestListFile>$(TestArchiveTestsDir)$(MSBuildProjectName).tests.list + <_TestListFileAbs>$([MSBuild]::NormalizePath('$(RepoRoot)', '$(_TestListFile)')) + <_MetadataFile>$(TestArchiveTestsDir)$(MSBuildProjectName).tests.metadata.json + <_MetadataFileAbs>$([MSBuild]::NormalizePath('$(RepoRoot)', '$(_MetadataFile)')) + + + <_RelativeProjectPath>$(MSBuildProjectDirectory.Replace('$(RepoRoot)', '')) + <_RelativeProjectPath>$(_RelativeProjectPath.Replace('\', '/')) + + + <_CollectionsToSkip Condition="'$(TestCollectionsToSkipSplitting)' != ''">$(TestCollectionsToSkipSplitting) + <_CollectionsToSkip Condition="'$(TestCollectionsToSkipSplitting)' == ''"> - - <_TestLines0 Include="$([System.Text.RegularExpressions.Regex]::Match('%(_ListOfTestsLines.Identity)', '$(_Regex)'))" /> - - + + + + - + <_InitialMetadataLines Include="{" /> + <_InitialMetadataLines Include=" "projectName": "$(MSBuildProjectName)"," /> + <_InitialMetadataLines Include=" "testClassNamesPrefix": "$(TestClassNamesPrefix)"," /> + <_InitialMetadataLines Include=" "testProjectPath": "$(_RelativeProjectPath)/$(MSBuildProjectFile)"," /> + <_InitialMetadataLines Include=" "requiresNugets": "$(RequiresNugetsForSplitTests)"," /> + <_InitialMetadataLines Include=" "requiresTestSdk": "$(RequiresTestSdkForSplitTests)"," /> + <_InitialMetadataLines Include=" "testSessionTimeout": "$(TestSessionTimeout)"," /> + <_InitialMetadataLines Include=" "testHangTimeout": "$(TestHangTimeout)"," /> + <_InitialMetadataLines Include=" "uncollectedTestsSessionTimeout": "$(UncollectedTestsSessionTimeout)"," /> + <_InitialMetadataLines Include=" "uncollectedTestsHangTimeout": "$(UncollectedTestsHangTimeout)"" /> + <_InitialMetadataLines Include="}" /> - - - - - - - - - - + + + + + <_PwshCommand>pwsh + <_PwshCommand Condition="'$(OS)' == 'Windows_NT'">powershell + <_TestAssemblyPath>$(TargetDir)$(TargetFileName) + <_DiscoveryCommand>$(_PwshCommand) -NoProfile -ExecutionPolicy Bypass -File "$(_DiscoveryScriptPath)" + <_DiscoveryCommand>$(_DiscoveryCommand) -TestAssemblyPath "$(_TestAssemblyPath)" + <_DiscoveryCommand>$(_DiscoveryCommand) -RunCommand "$(RunCommand)" + <_DiscoveryCommand>$(_DiscoveryCommand) -TestClassNamesPrefix "$(TestClassNamesPrefix)" + <_DiscoveryCommand Condition="'$(_CollectionsToSkip)' != ''">$(_DiscoveryCommand) -TestCollectionsToSkip "$(_CollectionsToSkip)" + <_DiscoveryCommand>$(_DiscoveryCommand) -OutputListFile "$(_TestListFileAbs)" + <_DiscoveryCommand>$(_DiscoveryCommand) -MetadataJsonFile "$(_MetadataFileAbs)" + <_DiscoveryCommand>$(_DiscoveryCommand) -RepoRoot "$(RepoRoot)" + + + + + + + + + diff --git a/tests/Shared/GetTestProjects.proj b/tests/Shared/GetTestProjects.proj deleted file mode 100644 index 1b280fdf065..00000000000 --- a/tests/Shared/GetTestProjects.proj +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - - - $(MSBuildThisFileDirectory)..\..\ - - - - <_TestProjectsToExclude Include="$(RepoRoot)tests\Shared\**\*Tests.csproj" /> - <_TestProjectsToExclude Include="$(RepoRoot)tests\testproject\**\*Tests.csproj" /> - <_TestProjectsToExclude Include="$(RepoRoot)tests\TestingAppHost1\**\*Tests.csproj" /> - - - <_TestProjectsToExclude Include="$(RepoRoot)tests\Aspire.EndToEnd.Tests\**\*Tests.csproj" /> - - <_TestProjectsToExclude Include="$(RepoRoot)tests\Aspire.Templates.Tests\**\*Tests.csproj" /> - - <_TestProjects Include="$(RepoRoot)tests\**\*Tests.csproj" - Exclude="@(_TestProjectsToExclude)" /> - - - - - - - - - - - - - - - - - - diff --git a/tests/helix/send-to-helix-templatestests.targets b/tests/helix/send-to-helix-templatestests.targets index 7a57735fb97..e84e9d5b1e4 100644 --- a/tests/helix/send-to-helix-templatestests.targets +++ b/tests/helix/send-to-helix-templatestests.targets @@ -32,9 +32,14 @@ - + + + + <_TemplateTestsClassNames Include="@(_TemplateTestsClassNamesRaw->'%(Identity)'->Replace('class:', ''))" /> + + diff --git a/tools/ExtractTestPartitions/ExtractTestPartitions.csproj b/tools/ExtractTestPartitions/ExtractTestPartitions.csproj new file mode 100644 index 00000000000..7cbfa621b34 --- /dev/null +++ b/tools/ExtractTestPartitions/ExtractTestPartitions.csproj @@ -0,0 +1,10 @@ + + + + Exe + $(DefaultTargetFramework) + enable + enable + + + diff --git a/tools/ExtractTestPartitions/Program.cs b/tools/ExtractTestPartitions/Program.cs new file mode 100644 index 00000000000..ab44615c8fd --- /dev/null +++ b/tools/ExtractTestPartitions/Program.cs @@ -0,0 +1,117 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Reflection; + +if (args.Length < 4 || args[0] != "--assembly-path" || args[2] != "--output-file") +{ + Console.Error.WriteLine("Usage: ExtractTestPartitions --assembly-path --output-file "); + return 1; +} + +var assemblyPath = args[1]; +var outputFile = args[3]; + +ExtractPartitions(assemblyPath, outputFile); +return 0; + +static void ExtractPartitions(string assemblyPath, string outputFile) +{ + if (!File.Exists(assemblyPath)) + { + Console.Error.WriteLine($"Error: Assembly file not found: {assemblyPath}"); + Environment.Exit(1); + } + + var partitions = new HashSet(StringComparer.OrdinalIgnoreCase); + + try + { + // Load the assembly using Assembly.LoadFrom + // We need to set up an assembly resolve handler for dependencies + var assemblyDirectory = Path.GetDirectoryName(assemblyPath)!; + AppDomain.CurrentDomain.AssemblyResolve += (sender, args) => + { + var assemblyName = new AssemblyName(args.Name); + var dllPath = Path.Combine(assemblyDirectory, assemblyName.Name + ".dll"); + if (File.Exists(dllPath)) + { + return Assembly.LoadFrom(dllPath); + } + return null; + }; + + var assembly = Assembly.LoadFrom(assemblyPath); + Console.WriteLine($"Loaded assembly: {assembly.FullName}"); + + // Iterate through all types in the assembly + Type[] types; + try + { + types = assembly.GetTypes(); + } + catch (ReflectionTypeLoadException ex) + { + // Some types couldn't be loaded due to missing dependencies + // Use the types that did load + types = ex.Types.Where(t => t != null).ToArray()!; + Console.WriteLine($"** Some types could not be loaded. Loaded {types.Length} types successfully."); + } + + foreach (var type in types) + { + // Check if type has Collection or Trait attributes + var attributes = type.GetCustomAttributesData(); + + foreach (var attr in attributes) + { + var attrTypeName = attr.AttributeType.FullName ?? attr.AttributeType.Name; + + if (!attrTypeName.EndsWith(".TraitAttribute") && attrTypeName != "TraitAttribute") + { + continue; + } + + if (attr.ConstructorArguments.Count < 2) + { + continue; + } + + var key = attr.ConstructorArguments[0].Value as string; + var value = attr.ConstructorArguments[1].Value as string; + + if (key?.Equals("Partition", StringComparison.OrdinalIgnoreCase) == true && + !string.IsNullOrWhiteSpace(value)) + { + partitions.Add(value); + Console.WriteLine($"Found Trait Partition: {value} on {type.Name}"); + } + } + } + + Console.WriteLine($"Total unique partitions found: {partitions.Count}"); + + // Write partitions to output file + var outputDir = Path.GetDirectoryName(outputFile); + if (!string.IsNullOrEmpty(outputDir) && !Directory.Exists(outputDir)) + { + Directory.CreateDirectory(outputDir); + } + + if (partitions.Count > 0) + { + File.WriteAllLines(outputFile, partitions.OrderBy(p => p, StringComparer.OrdinalIgnoreCase)); + Console.WriteLine($"Partitions written to: {outputFile}"); + } + else + { + Console.WriteLine("No partitions found. Not creating output file."); + } + } + catch (Exception ex) + { + Console.Error.WriteLine($"Error extracting partitions: {ex.Message}"); + Console.Error.WriteLine($"Stack trace: {ex.StackTrace}"); + Environment.Exit(1); + } +}