diff --git a/.claude/skills/quarantine-test/SKILL.md b/.claude/skills/quarantine-test/SKILL.md
new file mode 100644
index 00000000000..c2f9588d791
--- /dev/null
+++ b/.claude/skills/quarantine-test/SKILL.md
@@ -0,0 +1,228 @@
+---
+name: Quarantine Test
+description: Quarantine flaky or failing tests by adding the QuarantinedTest attribute using the QuarantineTools. Use when tests are failing intermittently or need to be excluded from regular test runs.
+allowed-tools: Read, Grep, Bash
+---
+
+# Quarantine Test
+
+## Purpose
+
+This skill quarantines flaky or failing tests by using the QuarantineTools utility to add the `[QuarantinedTest]` attribute. Quarantined tests are excluded from regular test runs and instead run in a separate quarantine workflow.
+
+## When to Use
+
+Invoke this skill when:
+- A test is failing intermittently (flaky)
+- A test doesn't fail deterministically
+- User requests to "quarantine a test"
+- A test needs to be temporarily excluded from CI while being fixed
+- You need to mark a test as quarantined with an associated GitHub issue
+
+## Important Context
+
+- Quarantined tests are marked with `[QuarantinedTest("issue-url")]` attribute
+- They are NOT run in the regular `tests.yml` workflow
+- They ARE run in the separate `tests-quarantine.yml` workflow every 6 hours
+- A GitHub issue URL is REQUIRED when quarantining tests
+- The QuarantineTools utility handles adding the attribute and managing using directives automatically
+
+## Instructions
+
+### Step 1: Identify the Test to Quarantine
+
+1. Get the fully-qualified test method name in format: `Namespace.ClassName.TestMethodName`
+2. If user provides partial name, use Grep to find the complete qualified name:
+ ```bash
+ grep -rn "void TestMethodName" tests/
+ ```
+3. Extract the namespace and class name from the file to build the fully-qualified name
+
+Example: `Aspire.Hosting.Tests.DistributedApplicationTests.TestMethodName`
+
+### Step 2: Get or Create GitHub Issue
+
+1. Check if the user provided a GitHub issue URL
+2. If not, ask: "What is the GitHub issue URL for tracking this flaky test?"
+3. The URL must be a valid http/https URL (e.g., `https://github.com/dotnet/aspire/issues/1234`)
+4. If no issue exists, suggest creating one first to track the test failure
+
+### Step 3: Run QuarantineTools
+
+Execute the QuarantineTools with the quarantine flag, test name(s), and issue URL:
+
+```bash
+dotnet run --project tools/QuarantineTools/QuarantineTools.csproj -- --quarantine "Namespace.ClassName.TestMethodName" --url "https://github.com/org/repo/issues/1234"
+```
+
+**Multiple tests** can be quarantined at once:
+```bash
+dotnet run --project tools/QuarantineTools/QuarantineTools.csproj -- --quarantine "Namespace.Class.Test1" "Namespace.Class.Test2" --url "https://github.com/org/repo/issues/1234"
+```
+
+**Command line flags:**
+- `-q` or `--quarantine`: Quarantine mode (add attribute)
+- `-i` or `--url`: GitHub issue URL (required for quarantine)
+- Tests: Fully-qualified method names (space-separated)
+
+### Step 4: Verify the Changes
+
+1. The tool will output which files were modified:
+ ```
+ Updated 1 file(s):
+ - tests/ProjectName.Tests/TestFile.cs
+ ```
+
+2. Read the modified file to confirm the attribute was added correctly:
+ ```bash
+ grep -A 2 -B 2 "QuarantinedTest" tests/ProjectName.Tests/TestFile.cs
+ ```
+
+3. Verify that:
+ - The `[QuarantinedTest("issue-url")]` attribute appears above the test method
+ - The `using Aspire.TestUtilities;` directive was added to the file (if not already present)
+
+### Step 5: Build and Run Tests to Confirm
+
+1. Build the test project to ensure no compilation errors:
+ ```bash
+ dotnet build tests/ProjectName.Tests/ProjectName.Tests.csproj
+ ```
+
+2. Run the test project with quarantine filter to verify the test is now quarantined:
+ ```bash
+ dotnet test tests/ProjectName.Tests/ProjectName.Tests.csproj --no-build -- --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true"
+ ```
+
+3. Confirm the quarantined test is NOT executed in the regular test run
+
+4. Optionally, verify the test CAN be run with the quarantine filter:
+ ```bash
+ dotnet test tests/ProjectName.Tests/ProjectName.Tests.csproj --no-build -- --filter-trait "quarantined=true"
+ ```
+
+### Step 6: Report Results
+
+Provide a clear summary:
+- Which test(s) were quarantined
+- The GitHub issue URL used
+- Which file(s) were modified
+- Confirmation that the test builds and is properly excluded from regular runs
+- Remind the user to commit the changes
+
+## Examples
+
+### Example 1: Quarantine a single flaky test
+
+User: "Quarantine the TestDistributedApplicationLifecycle test, it's flaky. Issue: https://github.com/dotnet/aspire/issues/5678"
+
+Actions:
+1. Find the fully-qualified name using Grep:
+ ```bash
+ grep -rn "void TestDistributedApplicationLifecycle" tests/
+ ```
+ Result: `tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs`
+
+2. Determine qualified name: `Aspire.Hosting.Tests.DistributedApplicationTests.TestDistributedApplicationLifecycle`
+
+3. Run QuarantineTools:
+ ```bash
+ dotnet run --project tools/QuarantineTools/QuarantineTools.csproj -- -q "Aspire.Hosting.Tests.DistributedApplicationTests.TestDistributedApplicationLifecycle" -i "https://github.com/dotnet/aspire/issues/5678"
+ ```
+
+4. Verify output shows file was updated
+
+5. Build and test:
+ ```bash
+ dotnet build tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj
+ dotnet test tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj --no-build -- --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true"
+ ```
+
+6. Report: "Quarantined test `TestDistributedApplicationLifecycle` with issue #5678. The test is now excluded from regular CI runs and will run in the quarantine workflow."
+
+### Example 2: Quarantine multiple related tests
+
+User: "These three tests in RedisTests are all flaky, quarantine them: TestRedisConnection, TestRedisCache, TestRedisCommands. Issue: https://github.com/dotnet/aspire/issues/9999"
+
+Actions:
+1. Find fully-qualified names (assuming they're in `Aspire.Components.Tests.RedisTests`)
+
+2. Run QuarantineTools with multiple tests:
+ ```bash
+ dotnet run --project tools/QuarantineTools/QuarantineTools.csproj -- -q "Aspire.Components.Tests.RedisTests.TestRedisConnection" "Aspire.Components.Tests.RedisTests.TestRedisCache" "Aspire.Components.Tests.RedisTests.TestRedisCommands" -i "https://github.com/dotnet/aspire/issues/9999"
+ ```
+
+3. Verify all three were modified
+
+4. Build and test the project
+
+5. Report: "Quarantined 3 tests from RedisTests with issue #9999."
+
+### Example 3: User provides short test name
+
+User: "Quarantine CanStartDashboard - it keeps timing out"
+
+Actions:
+1. Ask for GitHub issue: "What is the GitHub issue URL for tracking this flaky test?"
+
+2. User provides: "https://github.com/dotnet/aspire/issues/4321"
+
+3. Find the test:
+ ```bash
+ grep -rn "void CanStartDashboard" tests/
+ ```
+ Found in: `tests/Aspire.Dashboard.Tests/DashboardTests.cs`
+
+4. Read the file to determine namespace: `Aspire.Dashboard.Tests`
+
+5. Build fully-qualified name: `Aspire.Dashboard.Tests.DashboardTests.CanStartDashboard`
+
+6. Run QuarantineTools:
+ ```bash
+ dotnet run --project tools/QuarantineTools/QuarantineTools.csproj -- -q "Aspire.Dashboard.Tests.DashboardTests.CanStartDashboard" -i "https://github.com/dotnet/aspire/issues/4321"
+ ```
+
+7. Build and verify
+
+8. Report success
+
+## Common Issues and Troubleshooting
+
+### Issue: "No method found matching"
+- **Cause**: The fully-qualified name is incorrect
+- **Solution**: Use Grep to find the exact namespace, class name, and method name
+
+### Issue: "The test is already quarantined"
+- **Cause**: The attribute already exists on the method
+- **Solution**: Verify by reading the test file; no action needed
+
+### Issue: Tool reports "Quarantining requires a valid http(s) URL"
+- **Cause**: The issue URL is missing or malformed
+- **Solution**: Ensure the URL starts with `http://` or `https://`
+
+### Issue: Build fails after quarantining
+- **Cause**: The QuarantineTools may have encountered a syntax issue (rare)
+- **Solution**: Read the modified file and check for syntax errors; the tool should handle this correctly
+
+## Important Notes
+
+1. **Always build after quarantining** to ensure the changes are valid
+2. **Run tests to confirm** the quarantined test is properly excluded
+3. **Don't forget to commit** the modified test files
+4. **Track with GitHub issues** - every quarantined test should have an associated issue
+5. **The QuarantineTools handles**:
+ - Adding the `[QuarantinedTest("url")]` attribute
+ - Adding `using Aspire.TestUtilities;` if needed
+ - Preserving file formatting and indentation
+ - Supporting nested classes and various namespace styles
+
+## Unquarantining Tests
+
+To unquarantine a test (remove the attribute), use:
+```bash
+dotnet run --project tools/QuarantineTools/QuarantineTools.csproj -- -u "Namespace.ClassName.TestMethodName"
+```
+
+The tool will:
+- Remove the `[QuarantinedTest]` attribute
+- Remove the `using Aspire.TestUtilities;` directive if no other tests in the file use it
diff --git a/.claude/skills/test-runner/SKILL.md b/.claude/skills/test-runner/SKILL.md
new file mode 100644
index 00000000000..9a10437ecce
--- /dev/null
+++ b/.claude/skills/test-runner/SKILL.md
@@ -0,0 +1,166 @@
+---
+name: Aspire Test Runner
+description: Run tests for the Aspire project correctly, excluding quarantined and outerloop tests, with proper build verification. Use when running tests, debugging test failures, or validating changes.
+allowed-tools: Read, Grep, Glob, Bash
+---
+
+# Aspire Test Runner
+
+## Purpose
+
+This skill ensures tests are run correctly in the Aspire repository, following the project's specific requirements for test execution, including proper exclusion of quarantined and outerloop tests.
+
+## When to Use
+
+Invoke this skill when:
+- Running tests for a specific project or test class
+- Debugging test failures
+- Validating code changes
+- Verifying builds after modifications
+- User requests to "run tests" or "test my changes"
+
+## Critical Requirements
+
+**ALWAYS exclude quarantined and outerloop tests** in automated environments:
+- Quarantined tests are marked with `[QuarantinedTest]` and are known to be flaky
+- Outerloop tests are marked with `[OuterloopTest]` and are long-running or resource-intensive
+- These tests run separately in dedicated CI workflows
+
+## Instructions
+
+### Step 1: Identify Test Target
+
+1. If the user specifies a test project, use that path
+2. If the user mentions specific test methods or classes, identify the containing test project
+3. Use Glob to find test projects if needed:
+ ```bash
+ find tests -name "*.Tests.csproj" -type f
+ ```
+
+### Step 2: Build Verification (if needed)
+
+**Important**: Only build if:
+- There have been code changes since the last build
+- The user hasn't just run a successful build
+- You're unsure if the code is up to date
+
+If building is needed:
+```bash
+# Quick build with skip native (saves 1-2 minutes)
+./build.sh --build /p:SkipNativeBuild=true
+```
+
+### Step 3: Run Tests with Proper Filters
+
+**Default test run** (all tests in a project):
+```bash
+dotnet test tests/{ProjectName}.Tests/{ProjectName}.Tests.csproj --no-build -- --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true"
+```
+
+**Specific test method**:
+```bash
+dotnet test tests/{ProjectName}.Tests/{ProjectName}.Tests.csproj --no-build -- --filter-method "*.{MethodName}" --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true"
+```
+
+**Specific test class**:
+```bash
+dotnet test tests/{ProjectName}.Tests/{ProjectName}.Tests.csproj --no-build -- --filter-class "*.{ClassName}" --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true"
+```
+
+**Multiple test methods**:
+```bash
+dotnet test tests/{ProjectName}.Tests/{ProjectName}.Tests.csproj --no-build -- --filter-method "*.Method1" --filter-method "*.Method2" --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true"
+```
+
+### Step 4: Handle Test Results
+
+**If tests pass**:
+- Report success to the user
+- Mention the number of tests that passed
+
+**If tests fail**:
+1. Analyze the failure output
+2. Identify which tests failed and why
+3. Check if failures are related to recent code changes
+4. Suggest fixes or next steps
+5. Do NOT mark the task as complete if tests are failing
+
+**If snapshot tests fail**:
+1. Tests using Verify library will show snapshot differences
+2. After verifying the new output is correct, run:
+ ```bash
+ dotnet verify accept -y
+ ```
+3. Re-run the tests to confirm they pass
+
+### Step 5: Report Results
+
+Provide a clear summary:
+- Number of tests run
+- Pass/fail status
+- Any warnings or issues
+- Next steps if failures occurred
+
+## Examples
+
+### Example 1: Run all tests for a specific project
+
+User: "Run tests for Aspire.Hosting.Testing"
+
+Actions:
+1. Identify test project: `tests/Aspire.Hosting.Testing.Tests/Aspire.Hosting.Testing.Tests.csproj`
+2. Run with proper filters:
+ ```bash
+ dotnet test tests/Aspire.Hosting.Testing.Tests/Aspire.Hosting.Testing.Tests.csproj --no-build -- --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true"
+ ```
+3. Report results
+
+### Example 2: Run a specific test method
+
+User: "Run the TestingBuilderHasAllPropertiesFromRealBuilder test"
+
+Actions:
+1. Find the test using Grep:
+ ```bash
+ grep -r "TestingBuilderHasAllPropertiesFromRealBuilder" tests/
+ ```
+2. Identify project: `Aspire.Hosting.Testing.Tests`
+3. Run specific test:
+ ```bash
+ dotnet test tests/Aspire.Hosting.Testing.Tests/Aspire.Hosting.Testing.Tests.csproj --no-build -- --filter-method "*.TestingBuilderHasAllPropertiesFromRealBuilder" --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true"
+ ```
+4. Report results
+
+### Example 3: Run tests after making changes
+
+User: "I just modified the hosting code, run tests to verify"
+
+Actions:
+1. Identify affected test projects (e.g., `Aspire.Hosting.Tests`)
+2. Build first since code was modified:
+ ```bash
+ ./build.sh --build /p:SkipNativeBuild=true
+ ```
+3. Run tests:
+ ```bash
+ dotnet test tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj -- --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true"
+ ```
+ Note: No `--no-build` flag since we need to pick up the changes
+4. Report results
+
+## Common Pitfalls to Avoid
+
+1. **Never omit the quarantine and outerloop filters** - this will run flaky tests
+2. **Don't use `--no-build`** if code has changed - the changes won't be tested
+3. **Don't run the full test suite** - it takes 30+ minutes, use targeted testing
+4. **Don't ignore snapshot test failures** - they indicate output changes that need review
+5. **Don't forget the `--` separator** before filter arguments
+
+## Valid Test Filter Switches
+
+- `--filter-class` / `--filter-not-class`: Filter by class name
+- `--filter-method` / `--filter-not-method`: Filter by method name
+- `--filter-namespace` / `--filter-not-namespace`: Filter by namespace
+- `--filter-trait` / `--filter-not-trait`: Filter by trait (category, platform, etc.)
+
+Switches can be repeated to filter multiple values. Class and method filters expect fully qualified names, unless using a prefix like `*.ClassName`.
diff --git a/.github/actions/enumerate-tests/action.yml b/.github/actions/enumerate-tests/action.yml
index 3db315b0f17..d84fae85e61 100644
--- a/.github/actions/enumerate-tests/action.yml
+++ b/.github/actions/enumerate-tests/action.yml
@@ -11,12 +11,12 @@ inputs:
default: false
outputs:
- integrations_tests_matrix:
- description: Integration tests matrix
- value: ${{ steps.generate_integrations_matrix.outputs.integrations_tests_matrix }}
- templates_tests_matrix:
- description: Templates tests matrix
- value: ${{ steps.generate_templates_matrix.outputs.templates_tests_matrix }}
+ tests_matrix_requires_nugets:
+ description: Combined tests matrix for tests that require nugets
+ value: ${{ steps.generate_combined_matrix.outputs.tests_matrix_requires_nugets }}
+ tests_matrix_no_nugets:
+ description: Combined tests matrix for tests that do not require nugets
+ value: ${{ steps.generate_combined_matrix.outputs.tests_matrix_no_nugets }}
runs:
using: "composite"
steps:
@@ -28,60 +28,114 @@ runs:
with:
global-json-file: ${{ github.workspace }}/global.json
- - name: Get list of integration tests
- if: ${{ inputs.includeIntegrations }}
- shell: pwsh
- run: >
- dotnet build ${{ github.workspace }}/tests/Shared/GetTestProjects.proj
- /bl:${{ github.workspace }}/artifacts/log/Debug/GetTestProjects.binlog
- /p:TestsListOutputPath=${{ github.workspace }}/artifacts/TestsForGithubActions.list
- /p:ContinuousIntegrationBuild=true
+ - name: Restore
+ shell: bash
+ run: ./restore.sh
- - name: Generate list of template tests
- if: ${{ inputs.includeTemplates }}
- shell: pwsh
+ - name: Generate combined test matrix
+ shell: bash
run: >
- dotnet build ${{ github.workspace }}/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj
- "/t:Build;ExtractTestClassNames"
- /bl:${{ github.workspace }}/artifacts/log/Debug/BuildTemplatesTests.binlog
- -p:ExtractTestClassNamesForHelix=true
- -p:PrepareForHelix=true
- -p:ExtractTestClassNamesPrefix=Aspire.Templates.Tests
- -p:InstallBrowsersForPlaywright=false
-
- - name: Generate tests matrix
- id: generate_integrations_matrix
- if: ${{ inputs.includeIntegrations }}
- shell: pwsh
- run: |
- $filePath = "${{ github.workspace }}/artifacts/TestsForGithubActions.list"
- $lines = Get-Content $filePath
- $jsonObject = @{
- "shortname" = $lines | Sort-Object
- }
- $jsonString = ConvertTo-Json $jsonObject -Compress
- "integrations_tests_matrix=$jsonString"
- "integrations_tests_matrix=$jsonString" | Out-File -FilePath $env:GITHUB_OUTPUT
+ ./build.sh -test
+ /p:TestRunnerName=TestEnumerationRunsheetBuilder
+ /p:TestsListOutputPath=artifacts/TestsForGithubActions.list
+ /p:TestMatrixOutputPath=artifacts/combined-test-matrix.json
+ /p:BuildOs=${{ runner.os == 'Linux' && 'linux' || runner.os == 'macOS' && 'darwin' || 'windows' }}
+ -p:GenerateCIPartitions=true
+ -bl
- - name: Generate templates matrix
- id: generate_templates_matrix
- if: ${{ inputs.includeTemplates }}
+ - name: Generate combined matrix outputs
+ id: generate_combined_matrix
shell: pwsh
run: |
- $inputFilePath = "${{ github.workspace }}/artifacts/helix/templates-tests/Aspire.Templates.Tests.tests.list"
- $lines = Get-Content $inputFilePath
+ $matrixFilePath = "${{ github.workspace }}/artifacts/combined-test-matrix.json"
+ if (Test-Path $matrixFilePath) {
+ $matrixContent = Get-Content -Raw $matrixFilePath | ConvertFrom-Json
- $prefix = "Aspire.Templates.Tests."
- $lines = Get-Content $inputFilePath | ForEach-Object {
- $_ -replace "^$prefix", ""
- }
+ # Define defaults to apply when properties are missing
+ $defaults = @{
+ extraTestArgs = ''
+ requiresNugets = $false
+ requiresTestSdk = $false
+ testSessionTimeout = '20m'
+ testHangTimeout = '10m'
+ supportedOSes = @('windows', 'linux', 'macos')
+ }
+
+ # Split tests based on requiresNugets property and expand by OS
+ $testsRequiringNugets = @()
+ $testsNotRequiringNugets = @()
+
+ foreach ($test in $matrixContent.include) {
+ # Apply defaults for missing properties
+ foreach ($key in $defaults.Keys) {
+ if ($key -ne 'supportedOSes' -and -not $test.PSObject.Properties.Name.Contains($key)) {
+ $test | Add-Member -NotePropertyName $key -NotePropertyValue $defaults[$key] -Force
+ }
+ }
+
+ # Get supported OSes (use test-specific if present, otherwise default)
+ $supportedOSes = if ($test.PSObject.Properties.Name.Contains('supportedOSes')) {
+ $test.supportedOSes
+ } else {
+ $defaults.supportedOSes
+ }
+
+ # Expand test entry for each supported OS
+ foreach ($os in $supportedOSes) {
+ # Create a new entry for each OS
+ $testCopy = [PSCustomObject]@{}
+ foreach ($prop in $test.PSObject.Properties) {
+ if ($prop.Name -ne 'supportedOSes') {
+ $testCopy | Add-Member -NotePropertyName $prop.Name -NotePropertyValue $prop.Value -Force
+ }
+ }
+
+ # Add OS runner mapping
+ $osRunner = switch ($os) {
+ 'windows' { 'windows-latest' }
+ 'linux' { 'ubuntu-latest' }
+ 'macos' { 'macos-latest' }
+ default { 'ubuntu-latest' }
+ }
+ $testCopy | Add-Member -NotePropertyName 'runs-on' -NotePropertyValue $osRunner -Force
+
+ # Normalize boolean values to actual booleans (not strings)
+ if ($testCopy.PSObject.Properties.Name.Contains('requiresNugets')) {
+ $testCopy.requiresNugets = ($testCopy.requiresNugets -eq 'true' -or $testCopy.requiresNugets -eq $true)
+ }
+ if ($testCopy.PSObject.Properties.Name.Contains('requiresTestSdk')) {
+ $testCopy.requiresTestSdk = ($testCopy.requiresTestSdk -eq 'true' -or $testCopy.requiresTestSdk -eq $true)
+ }
+
+ # Add to appropriate list based on requiresNugets
+ if ($testCopy.requiresNugets -eq $true) {
+ $testsRequiringNugets += $testCopy
+ } else {
+ $testsNotRequiringNugets += $testCopy
+ }
+ }
+ }
+
+ # Create matrices
+ $nugetMatrix = @{ "include" = $testsRequiringNugets }
+ $nonNugetMatrix = @{ "include" = $testsNotRequiringNugets }
+
+ $nugetMatrixJson = ConvertTo-Json $nugetMatrix -Compress -Depth 10
+ $nonNugetMatrixJson = ConvertTo-Json $nonNugetMatrix -Compress -Depth 10
+
+ "tests_matrix_requires_nugets=$nugetMatrixJson" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
+ "tests_matrix_no_nugets=$nonNugetMatrixJson" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
- $jsonObject = @{
- "shortname" = $lines | Sort-Object
+ Write-Host "Combined test matrices generated successfully"
+ Write-Host "Tests requiring nugets: $($testsRequiringNugets.Count)"
+ Write-Host "Tests not requiring nugets: $($testsNotRequiringNugets.Count)"
+ } else {
+ # Empty matrices if no combined matrix found
+ $emptyMatrix = '{"include":[]}'
+ "tests_matrix_requires_nugets=$emptyMatrix" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
+ "tests_matrix_no_nugets=$emptyMatrix" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
+ Write-Host "No combined test matrix found, using empty matrices"
}
- $jsonString = ConvertTo-Json $jsonObject -Compress
- "templates_tests_matrix=$jsonString"
- "templates_tests_matrix=$jsonString" | Out-File -FilePath $env:GITHUB_OUTPUT
- name: Upload logs
if: always()
@@ -91,3 +145,4 @@ runs:
path: |
artifacts/log/**/*.binlog
artifacts/**/*.list
+ artifacts/combined-test-matrix.json
diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
index 3c366b56c41..e78871c1f7d 100644
--- a/.github/copilot-instructions.md
+++ b/.github/copilot-instructions.md
@@ -156,17 +156,6 @@ These switches can be repeated to run tests on multiple classes or methods at on
2. **Package Generation**: `./build.sh --pack` verifies all packages can be created
3. **Specific Tests**: Target individual test projects related to your changes
-## Quarantined tests
-
-- Tests that are flaky and don't fail deterministically are marked with the `QuarantinedTest` attribute.
-- Such tests are not run as part of the regular tests workflow (`tests.yml`).
- - Instead they are run in the `Quarantine` workflow (`tests-quarantine.yml`).
-- A github issue url is used with the attribute
-
-Example: `[QuarantinedTest("..issue url..")]`
-
-- To quarantine or unquarantine tests, use the tool in `tools/QuarantineTools/QuarantineTools.csproj`.
-
## Outerloop tests
- Tests that are long-running, resource-intensive, or require special infrastructure are marked with the `OuterloopTest` attribute.
@@ -194,8 +183,6 @@ The `*.Designer.cs` files are in the repo, but are intended to match same named
* Code blocks should be formatted with triple backticks (```) and include the language identifier for syntax highlighting.
* JSON code blocks should be indented properly.
-## Localization files
-* Files matching the pattern `*/localize/templatestrings.*.json` are localization files. Do not translate their content. It is done by a dedicated workflow.
## Trust These Instructions
These instructions are comprehensive and tested. Only search for additional information if:
diff --git a/.github/instructions/quarantine.instructions.md b/.github/instructions/quarantine.instructions.md
deleted file mode 100644
index 4d4952350ce..00000000000
--- a/.github/instructions/quarantine.instructions.md
+++ /dev/null
@@ -1,13 +0,0 @@
----
-applyTo: "tools/QuarantineTools/*"
----
-
-This tool is used to quarantine flaky tests.
-
-Usage:
-
-```bash
-dotnet run --project tools/QuarantineTools -- -q Namespace.Type.Method -i https://issue.url
-```
-
-Make sure to build the project containing the updated tests to ensure the changes don't break the build.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index a4cd05fa553..88cf2764ca1 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -9,45 +9,12 @@ on:
type: string
jobs:
- # Duplicated jobs so their dependencies are not blocked on both the
- # setup jobs
-
- setup_for_tests_lin:
- name: Setup for tests (Linux)
+ setup_for_tests:
+ name: Setup for tests
runs-on: ubuntu-latest
outputs:
- integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }}
- templates_tests_matrix: ${{ steps.generate_tests_matrix.outputs.templates_tests_matrix }}
- steps:
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-
- - uses: ./.github/actions/enumerate-tests
- id: generate_tests_matrix
- with:
- includeIntegrations: true
- includeTemplates: true
-
- setup_for_tests_macos:
- name: Setup for tests (macOS)
- runs-on: macos-latest
- outputs:
- integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }}
- templates_tests_matrix: ${{ steps.generate_tests_matrix.outputs.templates_tests_matrix }}
- steps:
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-
- - uses: ./.github/actions/enumerate-tests
- id: generate_tests_matrix
- with:
- includeIntegrations: true
- includeTemplates: true
-
- setup_for_tests_win:
- name: Setup for tests (Windows)
- runs-on: windows-latest
- outputs:
- integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }}
- templates_tests_matrix: ${{ steps.generate_tests_matrix.outputs.templates_tests_matrix }}
+ tests_matrix_requires_nugets: ${{ steps.generate_tests_matrix.outputs.tests_matrix_requires_nugets }}
+ tests_matrix_no_nugets: ${{ steps.generate_tests_matrix.outputs.tests_matrix_no_nugets }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -63,115 +30,43 @@ jobs:
with:
versionOverrideArg: ${{ inputs.versionOverrideArg }}
- integrations_test_lin:
- uses: ./.github/workflows/run-tests.yml
- name: Integrations Linux
- needs: setup_for_tests_lin
- strategy:
- fail-fast: false
- matrix:
- ${{ fromJson(needs.setup_for_tests_lin.outputs.integrations_tests_matrix) }}
- with:
- testShortName: ${{ matrix.shortname }}
- os: "ubuntu-latest"
- # Docker tests are run on linux, and Hosting tests take longer to finish
- testSessionTimeout: ${{ matrix.shortname == 'Hosting' && '25m' || '15m' }}
- extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\""
- versionOverrideArg: ${{ inputs.versionOverrideArg }}
-
- integrations_test_macos:
- uses: ./.github/workflows/run-tests.yml
- name: Integrations macos
- needs: setup_for_tests_macos
- strategy:
- fail-fast: false
- matrix:
- ${{ fromJson(needs.setup_for_tests_macos.outputs.integrations_tests_matrix) }}
- with:
- testShortName: ${{ matrix.shortname }}
- os: "macos-latest"
- extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\""
- versionOverrideArg: ${{ inputs.versionOverrideArg }}
-
- integrations_test_win:
+ tests_no_nugets:
uses: ./.github/workflows/run-tests.yml
- name: Integrations Windows
- needs: setup_for_tests_win
+ name: Tests
+ needs: setup_for_tests
+ if: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets).include[0] != null }}
strategy:
fail-fast: false
- matrix:
- ${{ fromJson(needs.setup_for_tests_win.outputs.integrations_tests_matrix) }}
+ matrix: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets) }}
with:
testShortName: ${{ matrix.shortname }}
- os: "windows-latest"
- extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\""
+ os: ${{ matrix.runs-on }}
+ testProjectPath: ${{ matrix.testProjectPath }}
+ testSessionTimeout: ${{ matrix.testSessionTimeout }}
+ testHangTimeout: ${{ matrix.testHangTimeout }}
+ extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}"
versionOverrideArg: ${{ inputs.versionOverrideArg }}
+ requiresNugets: ${{ matrix.requiresNugets }}
+ requiresTestSdk: ${{ matrix.requiresTestSdk }}
- templates_test_lin:
- name: Templates Linux
+ tests_requires_nugets:
+ name: Tests (Requires Nugets)
uses: ./.github/workflows/run-tests.yml
- needs: [setup_for_tests_lin, build_packages]
+ needs: [setup_for_tests, build_packages]
+ if: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets).include[0] != null }}
strategy:
fail-fast: false
- matrix: ${{ fromJson(needs.setup_for_tests_lin.outputs.templates_tests_matrix) }}
+ matrix: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets) }}
with:
testShortName: ${{ matrix.shortname }}
- os: "ubuntu-latest"
- testProjectPath: tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj
- testSessionTimeout: 20m
- testHangTimeout: 12m
- extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true --filter-class Aspire.Templates.Tests.${{ matrix.shortname }}"
- versionOverrideArg: ${{ inputs.versionOverrideArg }}
- requiresNugets: true
- requiresTestSdk: true
-
- templates_test_macos:
- name: Templates macos
- uses: ./.github/workflows/run-tests.yml
- needs: [setup_for_tests_macos, build_packages]
- strategy:
- fail-fast: false
- matrix: ${{ fromJson(needs.setup_for_tests_macos.outputs.templates_tests_matrix) }}
- with:
- testShortName: ${{ matrix.shortname }}
- os: "macos-latest"
- testProjectPath: tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj
- testSessionTimeout: 20m
- testHangTimeout: 12m
- extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true --filter-class Aspire.Templates.Tests.${{ matrix.shortname }}"
- versionOverrideArg: ${{ inputs.versionOverrideArg }}
- requiresNugets: true
- requiresTestSdk: true
-
- templates_test_win:
- name: Templates Windows
- uses: ./.github/workflows/run-tests.yml
- needs: [setup_for_tests_win, build_packages]
- strategy:
- fail-fast: false
- matrix: ${{ fromJson(needs.setup_for_tests_win.outputs.templates_tests_matrix) }}
- with:
- testShortName: ${{ matrix.shortname }}
- os: "windows-latest"
- testProjectPath: tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj
- testSessionTimeout: 20m
- testHangTimeout: 12m
- extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true --filter-class Aspire.Templates.Tests.${{ matrix.shortname }}"
- versionOverrideArg: ${{ inputs.versionOverrideArg }}
- requiresNugets: true
- requiresTestSdk: true
-
- endtoend_tests:
- name: EndToEnd Linux
- uses: ./.github/workflows/run-tests.yml
- needs: build_packages
- with:
- testShortName: EndToEnd
- # EndToEnd is not run on Windows/macOS due to missing Docker support
- os: ubuntu-latest
- testProjectPath: tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj
- requiresNugets: true
+ os: ${{ matrix.runs-on }}
+ testProjectPath: ${{ matrix.testProjectPath }}
+ testSessionTimeout: ${{ matrix.testSessionTimeout }}
+ testHangTimeout: ${{ matrix.testHangTimeout }}
+ extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}"
versionOverrideArg: ${{ inputs.versionOverrideArg }}
+ requiresNugets: ${{ matrix.requiresNugets }}
+ requiresTestSdk: ${{ matrix.requiresTestSdk }}
extension_tests_win:
name: Run VS Code extension tests (Windows)
@@ -185,7 +80,7 @@ jobs:
- name: Setup Node.js environment
uses: actions/setup-node@v2
with:
- node-version: ${{ matrix.node-version }}
+ node-version: '20'
- name: Install dependencies
run: yarn install
- name: Run tests
@@ -203,14 +98,9 @@ jobs:
runs-on: ubuntu-latest
name: Final Test Results
needs: [
- endtoend_tests,
extension_tests_win,
- integrations_test_lin,
- integrations_test_macos,
- integrations_test_win,
- templates_test_lin,
- templates_test_macos,
- templates_test_win
+ tests_no_nugets,
+ tests_requires_nugets
]
steps:
- name: Checkout code
diff --git a/eng/AfterSolutionBuild.targets b/eng/AfterSolutionBuild.targets
index 3288a591e3d..c391b2fa6ad 100644
--- a/eng/AfterSolutionBuild.targets
+++ b/eng/AfterSolutionBuild.targets
@@ -105,4 +105,71 @@
+
+
+
+
+
+
+
+
+ <_TestsListOutputPath Condition="'$(TestsListOutputPath)' != ''">$([MSBuild]::NormalizePath('$(RepoRoot)', '$(TestsListOutputPath)'))
+ <_TestsListOutputPath Condition="'$(TestsListOutputPath)' == ''">$(ArtifactsDir)/TestsForGithubActions.list
+ <_TestMatrixOutputPath Condition="'$(TestMatrixOutputPath)' != ''">$([MSBuild]::NormalizePath('$(RepoRoot)', '$(TestMatrixOutputPath)'))
+ <_BuildMatrixScript>$([MSBuild]::NormalizePath($(RepoRoot), 'eng', 'scripts', 'build-test-matrix.ps1'))
+
+
+ <_HelixDir>$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix'))
+
+
+ <_CurrentOS Condition="'$(BuildOs)' != ''">$(BuildOs)
+ <_CurrentOS Condition="'$(BuildOs)' == '' and $([MSBuild]::IsOSPlatform('Linux'))">all
+ <_CurrentOS Condition="'$(BuildOs)' == '' and $([MSBuild]::IsOSPlatform('Windows'))">all
+ <_CurrentOS Condition="'$(BuildOs)' == '' and $([MSBuild]::IsOSPlatform('OSX'))">all
+
+
+
+
+
+
+
+
diff --git a/eng/TestEnumerationRunsheetBuilder/DESIGN.md b/eng/TestEnumerationRunsheetBuilder/DESIGN.md
new file mode 100644
index 00000000000..9aae4375461
--- /dev/null
+++ b/eng/TestEnumerationRunsheetBuilder/DESIGN.md
@@ -0,0 +1,166 @@
+# TestEnumerationRunsheetBuilder Design
+
+## Overview
+
+This document describes the design for migrating the current `GetTestProjects.proj` test enumeration mechanism to work through the Arcade SDK's runsheet builder pattern.
+
+## Current Architecture
+
+### GetTestProjects.proj (Current)
+- **Approach**: Centralized test project discovery and enumeration
+- **Invocation**: Manual execution via `dotnet build tests/Shared/GetTestProjects.proj`
+- **Process**:
+ 1. Discovers all test projects using glob patterns
+ 2. Calls MSBuild on each project to determine GitHub Actions eligibility
+ 3. Builds split test projects to generate test class lists
+ 4. Generates final test lists and matrices using PowerShell scripts
+
+### Problems with Current Approach
+- Requires explicit invocation outside the standard build process
+- Not integrated with Arcade SDK's runsheet builder mechanism
+- Duplicates logic that could be shared with other runsheet builders
+
+## New Architecture: TestEnumerationRunsheetBuilder
+
+### Design Principles
+1. **Distributed Processing**: Each test project generates its own enumeration data during build
+2. **Arcade SDK Integration**: Follows the same pattern as existing runsheet builders
+3. **Reuse Existing Logic**: Leverages existing test enumeration and splitting mechanisms
+4. **Centralized Combination**: Final aggregation happens in `AfterSolutionBuild.targets`
+
+### Components
+
+#### 1. TestEnumerationRunsheetBuilder.targets
+- **Location**: `eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets`
+- **Purpose**: Runs once per test project to generate test enumeration data
+- **Outputs**: Per-project test enumeration files in `ArtifactsTmpDir`
+
+#### 2. Enhanced AfterSolutionBuild.targets
+- **Purpose**: Combines individual test enumeration files into final outputs
+- **Trigger**: When `TestRunnerName=TestEnumerationRunsheetBuilder`
+- **Outputs**: Same as current GetTestProjects.proj (test lists and matrices)
+
+### Flow Diagram
+
+```
+Build Process
+├── For each test project:
+│ ├── TestEnumerationRunsheetBuilder.targets runs
+│ ├── Generates project-specific enumeration data
+│ └── Writes to artifacts/tmp/{project}.testenumeration.json
+│
+└── After all projects built:
+ ├── AfterSolutionBuild.targets runs
+ ├── Collects all testenumeration.json files
+ ├── Processes split tests (if any)
+ └── Generates final outputs:
+ ├── TestsForGithubActions.list
+ ├── TestsForGithubActions.list.split-projects
+ └── test-matrices/*.json
+```
+
+## Implementation Details
+
+### TestEnumerationRunsheetBuilder.targets
+
+```msbuild
+
+
+
+
+
+
+
+
+
+ <_EnumerationData Include="{
+ 'project': '$(MSBuildProjectName)',
+ 'fullPath': '$(MSBuildProjectFullPath)',
+ 'shortName': '$(_ShortName)',
+ 'runOnGithubActions': '%(_ProjectInfo.RunTestsOnGithubActions)',
+ 'splitTests': '%(_ProjectInfo.SplitTests)'
+ }" />
+
+
+
+
+
+```
+
+### Enhanced AfterSolutionBuild.targets
+
+```msbuild
+
+
+
+
+ <_TestEnumerationFiles Include="$(ArtifactsTmpDir)/*.testenumeration.json" />
+
+
+
+
+ <_ProcessingScript>
+ # PowerShell script to:
+ # 1. Read all testenumeration.json files
+ # 2. Filter by OS and eligibility
+ # 3. Generate test lists and split test lists
+ # 4. Call existing matrix generation script for split tests
+
+
+
+
+
+
+```
+
+## Migration Strategy
+
+### Phase 1: Implementation
+1. Create `TestEnumerationRunsheetBuilder.targets`
+2. Enhance `AfterSolutionBuild.targets` with test enumeration logic
+3. Implement PowerShell processing script
+
+### Phase 2: Integration
+1. Update GitHub Actions workflows to use new approach
+2. Test compatibility with existing split test functionality
+3. Validate output format matches current GetTestProjects.proj
+
+### Phase 3: Cleanup
+1. Deprecate GetTestProjects.proj usage in workflows
+2. Remove manual invocation commands
+3. Document new usage pattern
+
+## Usage
+
+### Command Line
+```bash
+# Instead of manual GetTestProjects.proj invocation:
+dotnet build tests/Shared/GetTestProjects.proj /bl:artifacts/log/Debug/GetTestProjects.binlog /p:TestsListOutputPath=artifacts/TestsForGithubActions.list /p:TestMatrixOutputPath=artifacts/test-matrices/ /p:ContinuousIntegrationBuild=true /p:BuildOs=linux
+
+# New approach using runsheet builder:
+./build.cmd -test /p:TestRunnerName=TestEnumerationRunsheetBuilder /p:TestsListOutputPath=artifacts/TestsForGithubActions.list /p:TestMatrixOutputPath=artifacts/test-matrices/ /p:ContinuousIntegrationBuild=true /p:BuildOs=linux
+```
+
+### Integration with CI
+The new approach integrates seamlessly with the existing build infrastructure and requires minimal changes to GitHub Actions workflows.
+
+## Benefits
+
+1. **Consistency**: Follows the same pattern as other runsheet builders
+2. **Automatic Discovery**: No manual project enumeration required
+3. **Build Integration**: Leverages existing build process and caching
+4. **Maintainability**: Reduces code duplication and improves consistency
+5. **Extensibility**: Easy to add new test enumeration features
+
+## Backward Compatibility
+
+- Existing GetTestProjects.proj functionality remains unchanged
+- New approach generates identical output formats
+- Migration can be done incrementally per workflow
\ No newline at end of file
diff --git a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets
new file mode 100644
index 00000000000..caa87ccad9f
--- /dev/null
+++ b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets
@@ -0,0 +1,133 @@
+
+
+
+
+
+ <_ShouldSkipProject>false
+ <_ShouldSkipProject Condition="$(MSBuildProjectDirectory.Contains('tests\Shared'))">true
+ <_ShouldSkipProject Condition="$(MSBuildProjectDirectory.Contains('tests\testproject'))">true
+ <_ShouldSkipProject Condition="$(MSBuildProjectDirectory.Contains('tests\TestingAppHost1'))">true
+
+
+ <_ShortName>$([System.IO.Path]::GetFileNameWithoutExtension('$(MSBuildProjectName)').Replace('Aspire.', '').Replace('.Tests', ''))
+
+
+
+
+
+
+ <_CurrentProject Include="$(MSBuildProjectFullPath)" />
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <_RelativeProjectPath>$([System.String]::Copy('$(MSBuildProjectFullPath)').Replace('$(RepoRoot)', ''))
+
+ <_RelativeProjectPath Condition="$(_RelativeProjectPath.StartsWith('/')) or $(_RelativeProjectPath.StartsWith('\'))">$(_RelativeProjectPath.Substring(1))
+
+
+
+ <_HelixDir Condition="'$(TestArchiveTestsDir)' != ''">$(TestArchiveTestsDir)
+ <_HelixDir Condition="'$(TestArchiveTestsDir)' == ''">$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix'))
+ <_TestListFile>$(_HelixDir)$(MSBuildProjectName).tests.list
+ <_MetadataFile>$(_HelixDir)$(MSBuildProjectName).tests.metadata.json
+ <_HasTestMetadata Condition="Exists('$(_MetadataFile)')">true
+ <_HasTestMetadata Condition="'$(_HasTestMetadata)' != 'true'">false
+
+
+ <_OsArrayJson>%(_ProjectInfo.SupportedOSes)
+
+
+ <_EnumerationJson>{
+ "project": "$(MSBuildProjectName)",
+ "fullPath": "$(_RelativeProjectPath)",
+ "shortName": "$(_ShortName)",
+ "supportedOSes": [$(_OsArrayJson)],
+ "splitTests": "%(_ProjectInfo.SplitTests)",
+ "hasTestMetadata": "$(_HasTestMetadata)",
+ "testListFile": "$(_TestListFile.Replace('$(RepoRoot)', ''))",
+ "metadataFile": "$(_MetadataFile.Replace('$(RepoRoot)', ''))"
+}
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/eng/Testing.props b/eng/Testing.props
index d24ccae9410..5fad18f14ca 100644
--- a/eng/Testing.props
+++ b/eng/Testing.props
@@ -2,6 +2,12 @@
true
+
+ 20m
+ 10m
+ 15m
+ 10m
+
true
true
diff --git a/eng/Testing.targets b/eng/Testing.targets
index 853a3949d3b..e31f6bf0186 100644
--- a/eng/Testing.targets
+++ b/eng/Testing.targets
@@ -109,4 +109,143 @@
+
+
+
+ <_ShouldSplit>false
+ <_ShouldSplit Condition="'$(SplitTestsOnCI)' == 'true'">true
+
+
+
+
+ <_ProjectInfo Condition="'$(BuildOs)' == 'windows'"
+ Include="$(MSBuildProjectFullPath)"
+ RunTestsOnGithubActions="$(RunOnGithubActionsWindows)"
+ SplitTests="$(_ShouldSplit)"
+ MSBuildSourceProjectFile="$(MSBuildProjectFullPath)" />
+ <_ProjectInfo Condition="'$(BuildOs)' == 'linux'"
+ Include="$(MSBuildProjectFullPath)"
+ RunTestsOnGithubActions="$(RunOnGithubActionsLinux)"
+ SplitTests="$(_ShouldSplit)"
+ MSBuildSourceProjectFile="$(MSBuildProjectFullPath)" />
+ <_ProjectInfo Condition="'$(BuildOs)' == 'darwin'"
+ Include="$(MSBuildProjectFullPath)"
+ RunTestsOnGithubActions="$(RunOnGithubActionsMacOS)"
+ SplitTests="$(_ShouldSplit)"
+ MSBuildSourceProjectFile="$(MSBuildProjectFullPath)" />
+
+
+ <_ProjectInfo Condition="'$(BuildOs)' == ''"
+ Include="$(MSBuildProjectFullPath)"
+ RunTestsOnGithubActions="$(RunOnGithubActions)"
+ SplitTests="$(_ShouldSplit)"
+ MSBuildSourceProjectFile="$(MSBuildProjectFullPath)" />
+
+
+
+
+
+
+ <_ShouldSplit>false
+ <_ShouldSplit Condition="'$(SplitTestsOnCI)' == 'true'">true
+
+
+ <_SupportedOSes>
+ <_SupportedOSes Condition="'$(RunOnGithubActionsWindows)' == 'true'">$(_SupportedOSes)"windows",
+ <_SupportedOSes Condition="'$(RunOnGithubActionsLinux)' == 'true'">$(_SupportedOSes)"linux",
+ <_SupportedOSes Condition="'$(RunOnGithubActionsMacOS)' == 'true'">$(_SupportedOSes)"macos",
+
+ <_SupportedOSes Condition="'$(_SupportedOSes)' != ''">$(_SupportedOSes.TrimEnd(','))
+
+
+
+ <_ProjectInfo Include="$(MSBuildProjectFullPath)"
+ SupportedOSes="$(_SupportedOSes)"
+ SplitTests="$(_ShouldSplit)"
+ MSBuildSourceProjectFile="$(MSBuildProjectFullPath)" />
+
+
+
+
+
+
+
+ <_HelixDir Condition="'$(TestArchiveTestsDir)' != ''">$(TestArchiveTestsDir)
+ <_HelixDir Condition="'$(TestArchiveTestsDir)' == ''">$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix'))
+ <_MetadataFile>$(_HelixDir)$(MSBuildProjectName).tests.metadata.json
+
+
+ <_RelativeProjectPath>$([System.String]::Copy('$(MSBuildProjectFullPath)').Replace('$(RepoRoot)', ''))
+
+ <_RelativeProjectPath Condition="$(_RelativeProjectPath.StartsWith('/')) or $(_RelativeProjectPath.StartsWith('\\'))">$(_RelativeProjectPath.Substring(1))
+
+
+ <_RequiresNugets>false
+ <_RequiresNugets Condition="'$(RequiresNugetsForSplitTests)' == 'true'">true
+
+
+ <_RequiresTestSdk>false
+ <_RequiresTestSdk Condition="'$(RequiresTestSdkForSplitTests)' == 'true'">true
+
+
+
+ <_EnablePlaywrightInstall>false
+ <_EnablePlaywrightInstall Condition="'$(EnablePlaywrightInstallForSplitTests)' == 'true'">true
+
+
+ <_SupportedOSesJson>
+ <_SupportedOSesJson Condition="'$(RunOnGithubActionsWindows)' == 'true'">$(_SupportedOSesJson)"windows",
+ <_SupportedOSesJson Condition="'$(RunOnGithubActionsLinux)' == 'true'">$(_SupportedOSesJson)"linux",
+ <_SupportedOSesJson Condition="'$(RunOnGithubActionsMacOS)' == 'true'">$(_SupportedOSesJson)"macos",
+
+ <_SupportedOSesJson Condition="'$(_SupportedOSesJson)' != ''">$(_SupportedOSesJson.TrimEnd(','))
+
+
+ <_TestSessionTimeout Condition="'$(TestSessionTimeout)' != ''">$(TestSessionTimeout)
+ <_TestSessionTimeout Condition="'$(TestSessionTimeout)' == ''">20m
+ <_TestHangTimeout Condition="'$(TestHangTimeout)' != ''">$(TestHangTimeout)
+ <_TestHangTimeout Condition="'$(TestHangTimeout)' == ''">10m
+ <_UncollectedTestsSessionTimeout Condition="'$(UncollectedTestsSessionTimeout)' != ''">$(UncollectedTestsSessionTimeout)
+ <_UncollectedTestsSessionTimeout Condition="'$(UncollectedTestsSessionTimeout)' == ''">15m
+ <_UncollectedTestsHangTimeout Condition="'$(UncollectedTestsHangTimeout)' != ''">$(UncollectedTestsHangTimeout)
+ <_UncollectedTestsHangTimeout Condition="'$(UncollectedTestsHangTimeout)' == ''">10m
+
+
+
+ <_ShortName>$([System.IO.Path]::GetFileNameWithoutExtension('$(MSBuildProjectName)').Replace('Aspire.', '').Replace('.Tests', ''))
+
+
+ <_MetadataJson>{
+ "projectName": "$(MSBuildProjectName)",
+ "shortName": "$(_ShortName)",
+ "testClassNamesPrefix": "$(MSBuildProjectName)",
+ "testProjectPath": "$(_RelativeProjectPath)",
+ "requiresNugets": "$(_RequiresNugets.ToLowerInvariant())",
+ "requiresTestSdk": "$(_RequiresTestSdk.ToLowerInvariant())",
+ "enablePlaywrightInstall": "$(_EnablePlaywrightInstall.ToLowerInvariant())",
+ "supportedOSes": [$(_SupportedOSesJson)],
+ "testSessionTimeout": "$(_TestSessionTimeout)",
+ "testHangTimeout": "$(_TestHangTimeout)",
+ "uncollectedTestsSessionTimeout": "15m",
+ "uncollectedTestsHangTimeout": "10m"
+}
+
+
+
+
+
+
+
+
+
diff --git a/eng/scripts/build-test-matrix.ps1 b/eng/scripts/build-test-matrix.ps1
new file mode 100644
index 00000000000..9cef64e3179
--- /dev/null
+++ b/eng/scripts/build-test-matrix.ps1
@@ -0,0 +1,366 @@
+<#
+.SYNOPSIS
+ Builds the combined test matrix for GitHub Actions from test enumeration files.
+
+.DESCRIPTION
+ This script consolidates the functionality of process-test-enumeration.ps1 and
+ generate-test-matrix.ps1 into a single optimized script that:
+ 1. Collects all .testenumeration.json files
+ 2. Filters tests by supported OSes
+ 3. Separates regular tests from split tests
+ 4. Generates matrix entries for all tests (with partition/class splitting)
+ 5. Writes the final combined-tests-matrix.json in a single pass
+
+ No intermediate files are created - all data processing happens in memory.
+
+.PARAMETER ArtifactsTmpDir
+ Directory containing .testenumeration.json files from test projects.
+
+.PARAMETER ArtifactsHelixDir
+ Directory containing .tests.list and .tests.metadata.json files.
+
+.PARAMETER OutputMatrixFile
+ Path to write the combined test matrix JSON file.
+
+.PARAMETER TestsListOutputFile
+ Optional path to write backward-compatible test list file (regular tests only) used on AzDO
+
+.PARAMETER CurrentOS
+ Current operating system (linux, windows, macos). Filters tests by supported OSes.
+ If not specified or set to 'all', includes tests for all OSes without filtering.
+
+.NOTES
+ PowerShell 7+
+ Replaces: process-test-enumeration.ps1 + generate-test-matrix.ps1
+#>
+
+[CmdletBinding()]
+param(
+ [Parameter(Mandatory=$true)]
+ [string]$ArtifactsTmpDir,
+
+ [Parameter(Mandatory=$true)]
+ [string]$ArtifactsHelixDir,
+
+ [Parameter(Mandatory=$true)]
+ [string]$OutputMatrixFile,
+
+ [Parameter(Mandatory=$false)]
+ [string]$TestsListOutputFile = "",
+
+ [Parameter(Mandatory=$false)]
+ [string]$CurrentOS = "all"
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version Latest
+
+# Normalize OS name
+$CurrentOS = $CurrentOS.ToLowerInvariant()
+
+# Determine if we should filter by OS
+$filterByOS = $CurrentOS -ne 'all'
+
+if ($filterByOS) {
+ Write-Host "Building test matrix for OS: $CurrentOS"
+} else {
+ Write-Host "Building combined test matrix for all OSes"
+}
+Write-Host "Enumerations directory: $ArtifactsTmpDir"
+Write-Host "Helix directory: $ArtifactsHelixDir"
+
+# Helper function to create matrix entry for regular (non-split) tests
+function New-RegularTestEntry {
+ param(
+ [Parameter(Mandatory=$true)]
+ $Enumeration,
+ [Parameter(Mandatory=$false)]
+ $Metadata = $null
+ )
+
+ $entry = [ordered]@{
+ type = 'regular'
+ projectName = $Enumeration.project
+ name = $Enumeration.shortName
+ shortname = $Enumeration.shortName
+ testProjectPath = $Enumeration.fullPath
+ workitemprefix = $Enumeration.project
+ }
+
+ # Add metadata if available
+ if ($Metadata) {
+ if ($Metadata.PSObject.Properties['testSessionTimeout']) { $entry['testSessionTimeout'] = $Metadata.testSessionTimeout }
+ if ($Metadata.PSObject.Properties['testHangTimeout']) { $entry['testHangTimeout'] = $Metadata.testHangTimeout }
+ if ($Metadata.PSObject.Properties['requiresNugets'] -and $Metadata.requiresNugets -eq 'true') { $entry['requiresNugets'] = 'true' }
+ if ($Metadata.PSObject.Properties['requiresTestSdk'] -and $Metadata.requiresTestSdk -eq 'true') { $entry['requiresTestSdk'] = 'true' }
+ if ($Metadata.PSObject.Properties['extraTestArgs'] -and $Metadata.extraTestArgs) { $entry['extraTestArgs'] = $Metadata.extraTestArgs }
+ }
+
+ # Add supported OSes
+ $entry['supportedOSes'] = @($Enumeration.supportedOSes)
+
+ return $entry
+}
+
+# Helper function to create matrix entry for collection-based split tests
+function New-CollectionTestEntry {
+ param(
+ [Parameter(Mandatory=$true)]
+ [string]$CollectionName,
+ [Parameter(Mandatory=$true)]
+ $Metadata,
+ [Parameter(Mandatory=$true)]
+ [bool]$IsUncollected
+ )
+
+ $suffix = if ($IsUncollected) { 'uncollected' } else { $CollectionName }
+ $baseShortName = if ($Metadata.shortName) { $Metadata.shortName } else { $Metadata.projectName }
+
+ $entry = [ordered]@{
+ type = 'collection'
+ projectName = $Metadata.projectName
+ name = if ($IsUncollected) { $baseShortName } else { "$baseShortName-$suffix" }
+ shortname = if ($IsUncollected) { $baseShortName } else { "$baseShortName-$suffix" }
+ testProjectPath = $Metadata.testProjectPath
+ workitemprefix = "$($Metadata.projectName)_$suffix"
+ collection = $CollectionName
+ }
+
+ # Use uncollected timeouts if available, otherwise use regular
+ if ($IsUncollected) {
+ if ($Metadata.PSObject.Properties['uncollectedTestsSessionTimeout']) {
+ $entry['testSessionTimeout'] = $Metadata.uncollectedTestsSessionTimeout
+ } elseif ($Metadata.PSObject.Properties['testSessionTimeout']) {
+ $entry['testSessionTimeout'] = $Metadata.testSessionTimeout
+ }
+
+ if ($Metadata.PSObject.Properties['uncollectedTestsHangTimeout']) {
+ $entry['testHangTimeout'] = $Metadata.uncollectedTestsHangTimeout
+ } elseif ($Metadata.PSObject.Properties['testHangTimeout']) {
+ $entry['testHangTimeout'] = $Metadata.testHangTimeout
+ }
+ } else {
+ if ($Metadata.PSObject.Properties['testSessionTimeout']) { $entry['testSessionTimeout'] = $Metadata.testSessionTimeout }
+ if ($Metadata.PSObject.Properties['testHangTimeout']) { $entry['testHangTimeout'] = $Metadata.testHangTimeout }
+ }
+
+ if ($Metadata.PSObject.Properties['requiresNugets'] -and $Metadata.requiresNugets -eq 'true') { $entry['requiresNugets'] = 'true' }
+ if ($Metadata.PSObject.Properties['requiresTestSdk'] -and $Metadata.requiresTestSdk -eq 'true') { $entry['requiresTestSdk'] = 'true' }
+
+ # Add test filter for collection-based splitting
+ if ($IsUncollected) {
+ $entry['extraTestArgs'] = '--filter-not-trait "Partition=*"'
+ } else {
+ $entry['extraTestArgs'] = "--filter-trait `"Partition=$CollectionName`""
+ }
+
+ # Add supported OSes from metadata (should match enumeration)
+ if ($Metadata.PSObject.Properties['supportedOSes']) {
+ $entry['supportedOSes'] = @($Metadata.supportedOSes)
+ }
+
+ return $entry
+}
+
+# Helper function to create matrix entry for class-based split tests
+function New-ClassTestEntry {
+ param(
+ [Parameter(Mandatory=$true)]
+ [string]$ClassName,
+ [Parameter(Mandatory=$true)]
+ $Metadata
+ )
+
+ # Extract short class name (last segment after last dot)
+ $shortClassName = $ClassName.Split('.')[-1]
+ $baseShortName = if ($Metadata.shortName) { $Metadata.shortName } else { $Metadata.projectName }
+
+ $entry = [ordered]@{
+ type = 'class'
+ projectName = $Metadata.projectName
+ name = "$baseShortName-$shortClassName"
+ shortname = "$baseShortName-$shortClassName"
+ testProjectPath = $Metadata.testProjectPath
+ workitemprefix = "$($Metadata.projectName)_$shortClassName"
+ classname = $ClassName
+ }
+
+ if ($Metadata.PSObject.Properties['testSessionTimeout']) { $entry['testSessionTimeout'] = $Metadata.testSessionTimeout }
+ if ($Metadata.PSObject.Properties['testHangTimeout']) { $entry['testHangTimeout'] = $Metadata.testHangTimeout }
+ if ($Metadata.PSObject.Properties['requiresNugets'] -and $Metadata.requiresNugets -eq 'true') { $entry['requiresNugets'] = 'true' }
+ if ($Metadata.PSObject.Properties['requiresTestSdk'] -and $Metadata.requiresTestSdk -eq 'true') { $entry['requiresTestSdk'] = 'true' }
+
+ # Add test filter for class-based splitting
+ $entry['extraTestArgs'] = "--filter-class `"$ClassName`""
+
+ # Add supported OSes from metadata
+ if ($Metadata.PSObject.Properties['supportedOSes']) {
+ $entry['supportedOSes'] = @($Metadata.supportedOSes)
+ }
+
+ return $entry
+}
+
+# 1. Collect all enumeration files
+$enumerationFiles = @(Get-ChildItem -Path $ArtifactsTmpDir -Filter "*.testenumeration.json" -ErrorAction SilentlyContinue)
+
+if ($enumerationFiles.Count -eq 0) {
+ Write-Warning "No test enumeration files found in $ArtifactsTmpDir"
+ # Create empty matrix
+ $matrix = @{ include = @() }
+ $matrix | ConvertTo-Json -Depth 10 -Compress | Set-Content -Path $OutputMatrixFile
+ Write-Host "Created empty test matrix: $OutputMatrixFile"
+ exit 0
+}
+
+Write-Host "Found $($enumerationFiles.Count) test enumeration file(s)"
+
+# 2. Build matrix entries
+$matrixEntries = [System.Collections.Generic.List[object]]::new()
+$regularTestsList = [System.Collections.Generic.List[string]]::new()
+
+foreach ($enumFile in $enumerationFiles) {
+ $enum = Get-Content -Raw -Path $enumFile.FullName | ConvertFrom-Json
+
+ Write-Host "Processing: $($enum.project)"
+
+ # Filter by supported OSes (skip if current OS not supported)
+ # Only filter if a specific OS was requested
+ if ($filterByOS -and $enum.supportedOSes -and $enum.supportedOSes.Count -gt 0) {
+ $osSupported = $false
+ foreach ($os in $enum.supportedOSes) {
+ if ($os.ToLowerInvariant() -eq $CurrentOS) {
+ $osSupported = $true
+ break
+ }
+ }
+
+ if (-not $osSupported) {
+ Write-Host " ⊘ Skipping (not supported on $CurrentOS)"
+ continue
+ }
+ }
+
+ # Check if this is a split test with metadata
+ if ($enum.splitTests -eq 'true' -and $enum.hasTestMetadata -eq 'true') {
+ Write-Host " → Split test (processing partitions/classes)"
+
+ # Read metadata and test list - use paths from enumeration file if available
+ if ($enum.metadataFile) {
+ # Path is relative to repo root, make it absolute
+ $metaFile = Join-Path $PSScriptRoot "../../$($enum.metadataFile)" -Resolve -ErrorAction SilentlyContinue
+ if (-not $metaFile) {
+ $metaFile = [System.IO.Path]::Combine($PSScriptRoot, "../..", $enum.metadataFile)
+ }
+ } else {
+ $metaFile = Join-Path $ArtifactsHelixDir "$($enum.project).tests.metadata.json"
+ }
+
+ if ($enum.testListFile) {
+ # Path is relative to repo root, make it absolute
+ $listFile = Join-Path $PSScriptRoot "../../$($enum.testListFile)" -Resolve -ErrorAction SilentlyContinue
+ if (-not $listFile) {
+ $listFile = [System.IO.Path]::Combine($PSScriptRoot, "../..", $enum.testListFile)
+ }
+ } else {
+ $listFile = Join-Path $ArtifactsHelixDir "$($enum.project).tests.list"
+ }
+
+ if (-not (Test-Path $metaFile)) {
+ Write-Warning " ⚠ Metadata file not found: $metaFile"
+ continue
+ }
+
+ if (-not (Test-Path $listFile)) {
+ Write-Warning " ⚠ Test list file not found: $listFile"
+ continue
+ }
+
+ $metadata = Get-Content -Raw -Path $metaFile | ConvertFrom-Json
+
+ # Add supported OSes to metadata from enumeration
+ $metadata | Add-Member -Force -MemberType NoteProperty -Name 'supportedOSes' -Value $enum.supportedOSes
+
+ $listLines = Get-Content -Path $listFile
+
+ $partitionCount = 0
+ $classCount = 0
+
+ foreach ($line in $listLines) {
+ $line = $line.Trim()
+ if ([string]::IsNullOrWhiteSpace($line)) { continue }
+
+ if ($line -match '^collection:(.+)$') {
+ # Collection/partition entry
+ $collectionName = $Matches[1]
+ $entry = New-CollectionTestEntry -CollectionName $collectionName -Metadata $metadata -IsUncollected $false
+ $matrixEntries.Add($entry)
+ $partitionCount++
+ }
+ elseif ($line -match '^uncollected:\*$') {
+ # Uncollected tests entry
+ $entry = New-CollectionTestEntry -CollectionName '*' -Metadata $metadata -IsUncollected $true
+ $matrixEntries.Add($entry)
+ $partitionCount++
+ }
+ elseif ($line -match '^class:(.+)$') {
+ # Class-based entry
+ $className = $Matches[1]
+ $entry = New-ClassTestEntry -ClassName $className -Metadata $metadata
+ $matrixEntries.Add($entry)
+ $classCount++
+ }
+ }
+
+ Write-Host " ✓ Added $partitionCount partition(s) and $classCount class(es)"
+ }
+ else {
+ # Regular (non-split) test
+ #Write-Host " → Regular test"
+
+ # Try to load metadata if available - use path from enumeration file if available
+ if ($enum.metadataFile) {
+ # Path is relative to repo root, make it absolute
+ $metaFile = Join-Path $PSScriptRoot "../../$($enum.metadataFile)" -Resolve -ErrorAction SilentlyContinue
+ if (-not $metaFile) {
+ $metaFile = [System.IO.Path]::Combine($PSScriptRoot, "../..", $enum.metadataFile)
+ }
+ } else {
+ $metaFile = Join-Path $ArtifactsHelixDir "$($enum.project).tests.metadata.json"
+ }
+
+ $metadata = $null
+ if (Test-Path $metaFile) {
+ $metadata = Get-Content -Raw -Path $metaFile | ConvertFrom-Json
+ }
+
+ $entry = New-RegularTestEntry -Enumeration $enum -Metadata $metadata
+ $matrixEntries.Add($entry)
+ $regularTestsList.Add($enum.shortName)
+
+ #Write-Host " ✓ Added regular test"
+ }
+}
+
+# 3. Write final matrix
+Write-Host ""
+Write-Host "Generated $($matrixEntries.Count) total matrix entries"
+
+$matrix = @{ include = $matrixEntries }
+$outputDir = [System.IO.Path]::GetDirectoryName($OutputMatrixFile)
+if ($outputDir -and -not (Test-Path $outputDir)) {
+ New-Item -ItemType Directory -Path $outputDir -Force | Out-Null
+}
+
+$matrix | ConvertTo-Json -Depth 10 -Compress | Set-Content -Path $OutputMatrixFile -Encoding UTF8
+
+Write-Host "✓ Matrix written to: $OutputMatrixFile"
+
+# 4. Write backward-compatible test list if requested
+if ($TestsListOutputFile) {
+ $regularTestsList | Set-Content -Path $TestsListOutputFile -Encoding UTF8
+ Write-Host "✓ Test list written to: $TestsListOutputFile"
+}
+
+Write-Host ""
+Write-Host "Matrix build complete!"
diff --git a/eng/scripts/split-test-projects-for-ci.ps1 b/eng/scripts/split-test-projects-for-ci.ps1
new file mode 100644
index 00000000000..3b55314d01d
--- /dev/null
+++ b/eng/scripts/split-test-projects-for-ci.ps1
@@ -0,0 +1,213 @@
+<#
+.SYNOPSIS
+ Extract test metadata (collections or classes) from test assemblies.
+
+.DESCRIPTION
+ Determines splitting mode by extracting Collection and Trait attributes from the test assembly:
+ - Uses ExtractTestPartitions tool to find [Collection("name")] or [Trait("Partition", "name")] attributes
+ - If partitions found → partition mode (collections)
+ - Else → class mode (runs --list-tests to enumerate classes)
+ Outputs a .tests.list file with either:
+ collection:Name
+ ...
+ uncollected:* (always appended in collection mode)
+ OR
+ class:Full.Namespace.ClassName
+ ...
+
+ Also updates the per-project metadata JSON with mode and collections.
+
+.PARAMETER TestAssemblyPath
+ Path to the test assembly DLL for extracting partition attributes.
+
+.PARAMETER RunCommand
+ The command to run the test assembly (e.g., "dotnet exec ").
+ Only invoked if partition extraction fails and class-based splitting is needed.
+
+.PARAMETER TestClassNamesPrefix
+ Namespace prefix used to recognize test classes (e.g. Aspire.Templates.Tests).
+
+.PARAMETER TestCollectionsToSkip
+ Semicolon-separated collection names to exclude from dedicated jobs.
+
+.PARAMETER OutputListFile
+ Path to the .tests.list output file.
+
+.PARAMETER MetadataJsonFile
+ Path to the .tests.metadata.json file (script may append mode info).
+
+.PARAMETER RepoRoot
+ Path to the repository root (for locating the ExtractTestPartitions tool).
+
+.NOTES
+ PowerShell 7+
+ Fails fast if zero test classes discovered when in class mode.
+ Optimized to only run --list-tests when partition extraction fails.
+#>
+
+[CmdletBinding()]
+param(
+ [Parameter(Mandatory=$true)]
+ [string]$TestAssemblyPath,
+
+ [Parameter(Mandatory=$true)]
+ [string]$RunCommand,
+
+ [Parameter(Mandatory=$true)]
+ [string]$TestClassNamesPrefix,
+
+ [Parameter(Mandatory=$false)]
+ [string]$TestCollectionsToSkip = "",
+
+ [Parameter(Mandatory=$true)]
+ [string]$OutputListFile,
+
+ [Parameter(Mandatory=$false)]
+ [string]$MetadataJsonFile = "",
+
+ [Parameter(Mandatory=$true)]
+ [string]$RepoRoot
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version Latest
+
+if (-not (Test-Path $TestAssemblyPath)) {
+ Write-Error "TestAssemblyPath not found: $TestAssemblyPath"
+}
+
+$collections = [System.Collections.Generic.HashSet[string]]::new()
+$classes = [System.Collections.Generic.HashSet[string]]::new()
+
+# Extract partitions using the ExtractTestPartitions tool
+# This step is optional - if it fails, we'll fall back to class-based splitting
+$partitionsFile = Join-Path ([System.IO.Path]::GetTempPath()) "partitions-$([System.Guid]::NewGuid()).txt"
+try {
+ $toolPath = Join-Path $RepoRoot "artifacts/bin/ExtractTestPartitions/Debug/net8.0/ExtractTestPartitions.dll"
+
+ # Build the tool if it doesn't exist
+ if (-not (Test-Path $toolPath)) {
+ Write-Host "Building ExtractTestPartitions tool..."
+ $toolProjectPath = Join-Path $RepoRoot "tools/ExtractTestPartitions/ExtractTestPartitions.csproj"
+ & dotnet build $toolProjectPath -c Debug --nologo -v quiet
+ if ($LASTEXITCODE -ne 0) {
+ Write-Host "Warning: Failed to build ExtractTestPartitions tool. Using class-based splitting."
+ }
+ }
+
+ if (Test-Path $toolPath) {
+ Write-Host "Extracting partitions from assembly: $TestAssemblyPath"
+ $toolOutput = & dotnet $toolPath --assembly-path $TestAssemblyPath --output-file $partitionsFile 2>&1
+ $toolExitCode = $LASTEXITCODE
+
+ # Display tool output (informational)
+ if ($toolOutput) {
+ $toolOutput | Write-Host
+ }
+
+ # If partitions file was created, read it (even if exit code is non-zero)
+ if (Test-Path $partitionsFile) {
+ $partitionLines = Get-Content $partitionsFile -ErrorAction SilentlyContinue
+ if ($partitionLines) {
+ foreach ($partition in $partitionLines) {
+ if (-not [string]::IsNullOrWhiteSpace($partition)) {
+ $collections.Add($partition.Trim()) | Out-Null
+ }
+ }
+ Write-Host "Found $($collections.Count) partition(s) via attribute extraction"
+ }
+ }
+ elseif ($toolExitCode -ne 0) {
+ Write-Host "Partition extraction completed with warnings. Falling back to class-based splitting."
+ }
+ }
+} catch {
+ # Partition extraction is optional - if it fails, we fall back to class-based splitting
+ Write-Host "Partition extraction encountered an issue. Falling back to class-based splitting."
+ Write-Host "Details: $_"
+}
+finally {
+ # Clean up temp file
+ if (Test-Path $partitionsFile) {
+ Remove-Item $partitionsFile -ErrorAction SilentlyContinue
+ }
+}
+
+# Apply collection filtering
+$skipList = @()
+if ($TestCollectionsToSkip) {
+ $skipList = $TestCollectionsToSkip -split ';' | ForEach-Object { $_.Trim() } | Where-Object { $_ }
+}
+
+$filteredCollections = @($collections | Where-Object { $skipList -notcontains $_ })
+
+# Determine mode: if we have partitions, use collection mode; otherwise fall back to class mode
+$mode = if ($filteredCollections.Count -gt 0) { 'collection' } else { 'class' }
+
+# Only run --list-tests if we need class-based splitting (no partitions found)
+if ($mode -eq 'class') {
+ Write-Host "No partitions found. Running --list-tests to extract class names..."
+
+ # Run the test assembly with --list-tests to get all test names
+ $testOutput = & $RunCommand --filter-not-trait category=failing --list-tests 2>&1
+
+ if ($LASTEXITCODE -ne 0) {
+ Write-Warning "Test listing command failed with exit code $LASTEXITCODE. Attempting to parse partial output..."
+ }
+
+ # Extract class names from test listing
+ $classNamePattern = '^(\s*)' + [Regex]::Escape($TestClassNamesPrefix) + '\.([^\.]+)\.'
+
+ foreach ($line in $testOutput) {
+ $lineStr = $line.ToString()
+ # Extract class name from test name
+ # Format: " Namespace.ClassName.MethodName(...)" or "Namespace.ClassName.MethodName"
+ if ($lineStr -match $classNamePattern) {
+ $className = "$TestClassNamesPrefix.$($Matches[2])"
+ $classes.Add($className) | Out-Null
+ }
+ }
+
+ if ($classes.Count -eq 0) {
+ Write-Error "No test classes discovered matching prefix '$TestClassNamesPrefix'."
+ }
+}
+
+$outputDir = [System.IO.Path]::GetDirectoryName($OutputListFile)
+if ($outputDir -and -not (Test-Path $outputDir)) {
+ New-Item -ItemType Directory -Path $outputDir -Force | Out-Null
+}
+
+$lines = [System.Collections.Generic.List[string]]::new()
+
+if ($mode -eq 'collection') {
+ foreach ($c in ($filteredCollections | Sort-Object)) {
+ $lines.Add("collection:$c")
+ }
+ $lines.Add("uncollected:*")
+} else {
+ foreach ($cls in ($classes | Sort-Object)) {
+ $lines.Add("class:$cls")
+ }
+}
+
+$lines | Set-Content -Path $OutputListFile -Encoding UTF8
+
+if ($MetadataJsonFile -and (Test-Path $MetadataJsonFile)) {
+ try {
+ $meta = Get-Content -Raw -Path $MetadataJsonFile | ConvertFrom-Json
+ # Add or update properties
+ $meta | Add-Member -Force -MemberType NoteProperty -Name 'mode' -Value $mode
+ $meta | Add-Member -Force -MemberType NoteProperty -Name 'collections' -Value @($filteredCollections | Sort-Object)
+ $meta | Add-Member -Force -MemberType NoteProperty -Name 'classCount' -Value $classes.Count
+ $meta | Add-Member -Force -MemberType NoteProperty -Name 'collectionCount' -Value $filteredCollections.Count
+ $meta | ConvertTo-Json -Depth 20 | Set-Content -Path $MetadataJsonFile -Encoding UTF8
+ } catch {
+ Write-Warning "Failed updating metadata JSON: $_"
+ }
+}
+
+Write-Host "Mode: $mode"
+Write-Host "Collections discovered (after filtering): $($filteredCollections.Count)"
+Write-Host "Classes discovered: $($classes.Count)"
+Write-Host "Output list written: $OutputListFile"
diff --git a/tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj b/tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj
index ae826d9bb89..3ac5f69bbd1 100644
--- a/tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj
+++ b/tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj
@@ -17,6 +17,8 @@
false
testassets\testproject\
+ true
+
<_BuildForTestsRunningOutsideOfRepo Condition="'$(TestsRunningOutsideOfRepo)' == 'true' or '$(ContinuousIntegrationBuild)' == 'true'">true
$(_BuildForTestsRunningOutsideOfRepo)
BUILD_FOR_TESTS_RUNNING_OUTSIDE_OF_REPO;$(DefineConstants)
diff --git a/tests/Aspire.Hosting.Python.Tests/AddPythonAppTests.cs b/tests/Aspire.Hosting.Python.Tests/AddPythonAppTests.cs
index fbf4d2e3a44..ead38eb032f 100644
--- a/tests/Aspire.Hosting.Python.Tests/AddPythonAppTests.cs
+++ b/tests/Aspire.Hosting.Python.Tests/AddPythonAppTests.cs
@@ -1,4 +1,4 @@
-// Licensed to the .NET Foundation under one or more agreements.
+// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
#pragma warning disable CS0612
@@ -476,6 +476,7 @@ public void WithVirtualEnvironment_ThrowsOnNullOrEmptyPath()
}
[Fact]
+ [QuarantinedTest("https://foo.com/issue/123")]
public async Task WithVirtualEnvironment_CanBeChainedWithOtherExtensions()
{
using var builder = TestDistributedApplicationBuilder.Create().WithTestAndResourceLogging(outputHelper);
diff --git a/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj b/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj
index f55080a25d1..f5c00c87ed7 100644
--- a/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj
+++ b/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj
@@ -10,6 +10,16 @@
false
false
+
+
+ true
+ Aspire.Hosting.Tests
+
+
+ 30m
+ 15m
+ 20m
+ 10m
diff --git a/tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs b/tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs
index b8b75ce0b91..da8b961798b 100644
--- a/tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs
+++ b/tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs
@@ -29,6 +29,8 @@
namespace Aspire.Hosting.Tests;
+[Collection("DistributedApplicationTests")]
+[Trait("Partition", "DistributedApplicationTests")]
public class DistributedApplicationTests
{
private readonly ITestOutputHelper _testOutputHelper;
diff --git a/tests/Aspire.Hosting.Tests/MSBuildTests.cs b/tests/Aspire.Hosting.Tests/MSBuildTests.cs
index c1501234b68..d2072cb5506 100644
--- a/tests/Aspire.Hosting.Tests/MSBuildTests.cs
+++ b/tests/Aspire.Hosting.Tests/MSBuildTests.cs
@@ -6,6 +6,7 @@
namespace Aspire.Hosting.Tests;
+[Trait("Partition", "Partition1")]
public class MSBuildTests
{
///
diff --git a/tests/Aspire.Hosting.Tests/Publishing/ResourceContainerImageBuilderTests.cs b/tests/Aspire.Hosting.Tests/Publishing/ResourceContainerImageBuilderTests.cs
index 7af0a8d3306..48e9a7c722f 100644
--- a/tests/Aspire.Hosting.Tests/Publishing/ResourceContainerImageBuilderTests.cs
+++ b/tests/Aspire.Hosting.Tests/Publishing/ResourceContainerImageBuilderTests.cs
@@ -13,6 +13,7 @@
namespace Aspire.Hosting.Tests.Publishing;
+[Trait("Partition", "Partition1")]
public class ResourceContainerImageBuilderTests(ITestOutputHelper output)
{
[Fact]
@@ -803,15 +804,15 @@ public async Task ResolveValue_FormatsDecimalWithInvariantCulture()
// Test decimal value
var result = await ResourceContainerImageBuilder.ResolveValue(3.14, CancellationToken.None);
Assert.Equal("3.14", result);
-
+
// Test double value
result = await ResourceContainerImageBuilder.ResolveValue(3.14d, CancellationToken.None);
Assert.Equal("3.14", result);
-
+
// Test float value
result = await ResourceContainerImageBuilder.ResolveValue(3.14f, CancellationToken.None);
Assert.Equal("3.14", result);
-
+
// Test integer (should also work)
result = await ResourceContainerImageBuilder.ResolveValue(42, CancellationToken.None);
Assert.Equal("42", result);
diff --git a/tests/Aspire.Hosting.Tests/WaitForTests.cs b/tests/Aspire.Hosting.Tests/WaitForTests.cs
index 559d8f703d8..e6639bbafab 100644
--- a/tests/Aspire.Hosting.Tests/WaitForTests.cs
+++ b/tests/Aspire.Hosting.Tests/WaitForTests.cs
@@ -11,6 +11,8 @@
namespace Aspire.Hosting.Tests;
+[Collection("WaitForTests")]
+[Trait("Partition", "WaitForTests")]
public class WaitForTests(ITestOutputHelper testOutputHelper)
{
[Fact]
diff --git a/tests/Aspire.Hosting.Tests/WithUrlsTests.cs b/tests/Aspire.Hosting.Tests/WithUrlsTests.cs
index ce1a3ce9842..29fb142e585 100644
--- a/tests/Aspire.Hosting.Tests/WithUrlsTests.cs
+++ b/tests/Aspire.Hosting.Tests/WithUrlsTests.cs
@@ -11,6 +11,7 @@
namespace Aspire.Hosting.Tests;
+[Trait("Partition", "Partition1")]
public class WithUrlsTests
{
[Fact]
diff --git a/tests/Aspire.Playground.Tests/Aspire.Playground.Tests.csproj b/tests/Aspire.Playground.Tests/Aspire.Playground.Tests.csproj
index 6b668f0186a..b52441b8353 100644
--- a/tests/Aspire.Playground.Tests/Aspire.Playground.Tests.csproj
+++ b/tests/Aspire.Playground.Tests/Aspire.Playground.Tests.csproj
@@ -1,4 +1,4 @@
-
+
$(DefaultTargetFramework)
@@ -105,6 +105,7 @@
+
diff --git a/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj b/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj
index cfd44eedfd5..31aca3598d5 100644
--- a/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj
+++ b/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj
@@ -13,8 +13,18 @@
true
true
- true
- Aspire.Templates.Tests
+ true
+ Aspire.Templates.Tests
+
+
+
+ true
+ true
+ true
+
+
+ 20m
+ 15m
-
+
-
+
-
-
-
+
- <_Regex>^\s*($(ExtractTestClassNamesPrefix)[^\($]+)
+
+ <_DiscoveryScriptPath>$(RepoRoot)eng\scripts\split-test-projects-for-ci.ps1
+
+
+ <_TestListFile>$(TestArchiveTestsDir)$(MSBuildProjectName).tests.list
+ <_TestListFileAbs>$([MSBuild]::NormalizePath('$(RepoRoot)', '$(_TestListFile)'))
+ <_MetadataFile>$(TestArchiveTestsDir)$(MSBuildProjectName).tests.metadata.json
+ <_MetadataFileAbs>$([MSBuild]::NormalizePath('$(RepoRoot)', '$(_MetadataFile)'))
+
+
+ <_RelativeProjectPath>$(MSBuildProjectDirectory.Replace('$(RepoRoot)', ''))
+ <_RelativeProjectPath>$(_RelativeProjectPath.Replace('\', '/'))
+
+
+ <_CollectionsToSkip Condition="'$(TestCollectionsToSkipSplitting)' != ''">$(TestCollectionsToSkipSplitting)
+ <_CollectionsToSkip Condition="'$(TestCollectionsToSkipSplitting)' == ''">
-
- <_TestLines0 Include="$([System.Text.RegularExpressions.Regex]::Match('%(_ListOfTestsLines.Identity)', '$(_Regex)'))" />
-
-
+
+
+
+
-
+ <_InitialMetadataLines Include="{" />
+ <_InitialMetadataLines Include=" "projectName": "$(MSBuildProjectName)"," />
+ <_InitialMetadataLines Include=" "testClassNamesPrefix": "$(TestClassNamesPrefix)"," />
+ <_InitialMetadataLines Include=" "testProjectPath": "$(_RelativeProjectPath)/$(MSBuildProjectFile)"," />
+ <_InitialMetadataLines Include=" "requiresNugets": "$(RequiresNugetsForSplitTests)"," />
+ <_InitialMetadataLines Include=" "requiresTestSdk": "$(RequiresTestSdkForSplitTests)"," />
+ <_InitialMetadataLines Include=" "testSessionTimeout": "$(TestSessionTimeout)"," />
+ <_InitialMetadataLines Include=" "testHangTimeout": "$(TestHangTimeout)"," />
+ <_InitialMetadataLines Include=" "uncollectedTestsSessionTimeout": "$(UncollectedTestsSessionTimeout)"," />
+ <_InitialMetadataLines Include=" "uncollectedTestsHangTimeout": "$(UncollectedTestsHangTimeout)"" />
+ <_InitialMetadataLines Include="}" />
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+ <_PwshCommand>pwsh
+ <_PwshCommand Condition="'$(OS)' == 'Windows_NT'">powershell
+ <_TestAssemblyPath>$(TargetDir)$(TargetFileName)
+ <_DiscoveryCommand>$(_PwshCommand) -NoProfile -ExecutionPolicy Bypass -File "$(_DiscoveryScriptPath)"
+ <_DiscoveryCommand>$(_DiscoveryCommand) -TestAssemblyPath "$(_TestAssemblyPath)"
+ <_DiscoveryCommand>$(_DiscoveryCommand) -RunCommand "$(RunCommand)"
+ <_DiscoveryCommand>$(_DiscoveryCommand) -TestClassNamesPrefix "$(TestClassNamesPrefix)"
+ <_DiscoveryCommand Condition="'$(_CollectionsToSkip)' != ''">$(_DiscoveryCommand) -TestCollectionsToSkip "$(_CollectionsToSkip)"
+ <_DiscoveryCommand>$(_DiscoveryCommand) -OutputListFile "$(_TestListFileAbs)"
+ <_DiscoveryCommand>$(_DiscoveryCommand) -MetadataJsonFile "$(_MetadataFileAbs)"
+ <_DiscoveryCommand>$(_DiscoveryCommand) -RepoRoot "$(RepoRoot)"
+
+
+
+
+
+
+
+
+
diff --git a/tests/Shared/GetTestProjects.proj b/tests/Shared/GetTestProjects.proj
deleted file mode 100644
index 1b280fdf065..00000000000
--- a/tests/Shared/GetTestProjects.proj
+++ /dev/null
@@ -1,48 +0,0 @@
-
-
-
-
-
-
-
- $(MSBuildThisFileDirectory)..\..\
-
-
-
- <_TestProjectsToExclude Include="$(RepoRoot)tests\Shared\**\*Tests.csproj" />
- <_TestProjectsToExclude Include="$(RepoRoot)tests\testproject\**\*Tests.csproj" />
- <_TestProjectsToExclude Include="$(RepoRoot)tests\TestingAppHost1\**\*Tests.csproj" />
-
-
- <_TestProjectsToExclude Include="$(RepoRoot)tests\Aspire.EndToEnd.Tests\**\*Tests.csproj" />
-
- <_TestProjectsToExclude Include="$(RepoRoot)tests\Aspire.Templates.Tests\**\*Tests.csproj" />
-
- <_TestProjects Include="$(RepoRoot)tests\**\*Tests.csproj"
- Exclude="@(_TestProjectsToExclude)" />
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/tests/helix/send-to-helix-templatestests.targets b/tests/helix/send-to-helix-templatestests.targets
index 7a57735fb97..e84e9d5b1e4 100644
--- a/tests/helix/send-to-helix-templatestests.targets
+++ b/tests/helix/send-to-helix-templatestests.targets
@@ -32,9 +32,14 @@
-
+
+
+
+ <_TemplateTestsClassNames Include="@(_TemplateTestsClassNamesRaw->'%(Identity)'->Replace('class:', ''))" />
+
+
diff --git a/tools/ExtractTestPartitions/ExtractTestPartitions.csproj b/tools/ExtractTestPartitions/ExtractTestPartitions.csproj
new file mode 100644
index 00000000000..7cbfa621b34
--- /dev/null
+++ b/tools/ExtractTestPartitions/ExtractTestPartitions.csproj
@@ -0,0 +1,10 @@
+
+
+
+ Exe
+ $(DefaultTargetFramework)
+ enable
+ enable
+
+
+
diff --git a/tools/ExtractTestPartitions/Program.cs b/tools/ExtractTestPartitions/Program.cs
new file mode 100644
index 00000000000..ab44615c8fd
--- /dev/null
+++ b/tools/ExtractTestPartitions/Program.cs
@@ -0,0 +1,117 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Reflection;
+
+if (args.Length < 4 || args[0] != "--assembly-path" || args[2] != "--output-file")
+{
+ Console.Error.WriteLine("Usage: ExtractTestPartitions --assembly-path --output-file ");
+ return 1;
+}
+
+var assemblyPath = args[1];
+var outputFile = args[3];
+
+ExtractPartitions(assemblyPath, outputFile);
+return 0;
+
+static void ExtractPartitions(string assemblyPath, string outputFile)
+{
+ if (!File.Exists(assemblyPath))
+ {
+ Console.Error.WriteLine($"Error: Assembly file not found: {assemblyPath}");
+ Environment.Exit(1);
+ }
+
+ var partitions = new HashSet(StringComparer.OrdinalIgnoreCase);
+
+ try
+ {
+ // Load the assembly using Assembly.LoadFrom
+ // We need to set up an assembly resolve handler for dependencies
+ var assemblyDirectory = Path.GetDirectoryName(assemblyPath)!;
+ AppDomain.CurrentDomain.AssemblyResolve += (sender, args) =>
+ {
+ var assemblyName = new AssemblyName(args.Name);
+ var dllPath = Path.Combine(assemblyDirectory, assemblyName.Name + ".dll");
+ if (File.Exists(dllPath))
+ {
+ return Assembly.LoadFrom(dllPath);
+ }
+ return null;
+ };
+
+ var assembly = Assembly.LoadFrom(assemblyPath);
+ Console.WriteLine($"Loaded assembly: {assembly.FullName}");
+
+ // Iterate through all types in the assembly
+ Type[] types;
+ try
+ {
+ types = assembly.GetTypes();
+ }
+ catch (ReflectionTypeLoadException ex)
+ {
+ // Some types couldn't be loaded due to missing dependencies
+ // Use the types that did load
+ types = ex.Types.Where(t => t != null).ToArray()!;
+ Console.WriteLine($"** Some types could not be loaded. Loaded {types.Length} types successfully.");
+ }
+
+ foreach (var type in types)
+ {
+ // Check if type has Collection or Trait attributes
+ var attributes = type.GetCustomAttributesData();
+
+ foreach (var attr in attributes)
+ {
+ var attrTypeName = attr.AttributeType.FullName ?? attr.AttributeType.Name;
+
+ if (!attrTypeName.EndsWith(".TraitAttribute") && attrTypeName != "TraitAttribute")
+ {
+ continue;
+ }
+
+ if (attr.ConstructorArguments.Count < 2)
+ {
+ continue;
+ }
+
+ var key = attr.ConstructorArguments[0].Value as string;
+ var value = attr.ConstructorArguments[1].Value as string;
+
+ if (key?.Equals("Partition", StringComparison.OrdinalIgnoreCase) == true &&
+ !string.IsNullOrWhiteSpace(value))
+ {
+ partitions.Add(value);
+ Console.WriteLine($"Found Trait Partition: {value} on {type.Name}");
+ }
+ }
+ }
+
+ Console.WriteLine($"Total unique partitions found: {partitions.Count}");
+
+ // Write partitions to output file
+ var outputDir = Path.GetDirectoryName(outputFile);
+ if (!string.IsNullOrEmpty(outputDir) && !Directory.Exists(outputDir))
+ {
+ Directory.CreateDirectory(outputDir);
+ }
+
+ if (partitions.Count > 0)
+ {
+ File.WriteAllLines(outputFile, partitions.OrderBy(p => p, StringComparer.OrdinalIgnoreCase));
+ Console.WriteLine($"Partitions written to: {outputFile}");
+ }
+ else
+ {
+ Console.WriteLine("No partitions found. Not creating output file.");
+ }
+ }
+ catch (Exception ex)
+ {
+ Console.Error.WriteLine($"Error extracting partitions: {ex.Message}");
+ Console.Error.WriteLine($"Stack trace: {ex.StackTrace}");
+ Environment.Exit(1);
+ }
+}