From 05653263ad9a03725733c412f2a90493529cacdf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 16 Oct 2025 19:40:19 +0000 Subject: [PATCH 01/48] Initial plan From 4c3edacd0ec23eb4851ca2aabbe1bb2ee42d450e Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Thu, 16 Oct 2025 15:51:42 -0400 Subject: [PATCH 02/48] implement plans and details --- docs/test-splitting/IMPLEMENTATION_PLAN.md | 203 ++++ docs/test-splitting/IMPLEMENTATION_PLAN_V2.md | 384 ++++++++ docs/test-splitting/IMPLEMENTATION_PLAN_V3.md | 332 +++++++ docs/test-splitting/IMPLEMENTATION_SUMMARY.md | 255 ++++++ .../STEP_01_DISCOVERY_HELPER.md | 368 ++++++++ .../test-splitting/STEP_01_MSBUILD_TARGETS.md | 236 +++++ .../STEP_01_MSBUILD_TARGETS_V2.md | 295 ++++++ .../STEP_02_MSBUILD_TARGETS_V3.md | 446 +++++++++ .../STEP_02_POWERSHELL_SCRIPT.md | 374 ++++++++ .../STEP_02_POWERSHELL_SCRIPT_V2.md | 708 ++++++++++++++ docs/test-splitting/STEP_03_GITHUB_ACTIONS.md | 414 +++++++++ .../STEP_03_MATRIX_GENERATOR_V3.md | 865 ++++++++++++++++++ docs/test-splitting/STEP_04_PROJECT_CONFIG.md | 230 +++++ .../STEP_04_PROJECT_CONFIG_V2.md | 490 ++++++++++ .../STEP_04_PROJECT_CONFIG_V3.md | 316 +++++++ docs/test-splitting/STEP_05_TESTING_V3.md | 373 ++++++++ docs/test-splitting/STEP_06_CI_INTEGRATION.md | 318 +++++++ eng/scripts/extract-test-metadata.ps1 | 136 +++ eng/scripts/generate-test-matrix.ps1 | 186 ++++ 19 files changed, 6929 insertions(+) create mode 100644 docs/test-splitting/IMPLEMENTATION_PLAN.md create mode 100644 docs/test-splitting/IMPLEMENTATION_PLAN_V2.md create mode 100644 docs/test-splitting/IMPLEMENTATION_PLAN_V3.md create mode 100644 docs/test-splitting/IMPLEMENTATION_SUMMARY.md create mode 100644 docs/test-splitting/STEP_01_DISCOVERY_HELPER.md create mode 100644 docs/test-splitting/STEP_01_MSBUILD_TARGETS.md create mode 100644 docs/test-splitting/STEP_01_MSBUILD_TARGETS_V2.md create mode 100644 docs/test-splitting/STEP_02_MSBUILD_TARGETS_V3.md create mode 100644 docs/test-splitting/STEP_02_POWERSHELL_SCRIPT.md create mode 100644 docs/test-splitting/STEP_02_POWERSHELL_SCRIPT_V2.md create mode 100644 docs/test-splitting/STEP_03_GITHUB_ACTIONS.md create mode 100644 docs/test-splitting/STEP_03_MATRIX_GENERATOR_V3.md create mode 100644 docs/test-splitting/STEP_04_PROJECT_CONFIG.md create mode 100644 docs/test-splitting/STEP_04_PROJECT_CONFIG_V2.md create mode 100644 docs/test-splitting/STEP_04_PROJECT_CONFIG_V3.md create mode 100644 docs/test-splitting/STEP_05_TESTING_V3.md create mode 100644 docs/test-splitting/STEP_06_CI_INTEGRATION.md create mode 100644 eng/scripts/extract-test-metadata.ps1 create mode 100644 eng/scripts/generate-test-matrix.ps1 diff --git a/docs/test-splitting/IMPLEMENTATION_PLAN.md b/docs/test-splitting/IMPLEMENTATION_PLAN.md new file mode 100644 index 00000000000..a66f75d464d --- /dev/null +++ b/docs/test-splitting/IMPLEMENTATION_PLAN.md @@ -0,0 +1,203 @@ +# Test Splitting Implementation Plan for dotnet/aspire + +**Date**: 2025-10-16 +**Author**: @radical +**Objective**: Implement a unified, MSBuild-based test splitting mechanism that works across all 3 OSes (Linux, macOS, Windows) and both CI systems (GitHub Actions, Azure DevOps). + +## Overview + +This plan implements automatic test partitioning by class for long-running test projects. The mechanism: +- ✅ Works on all 3 OSes (Linux, macOS, Windows) +- ✅ Works on GitHub Actions and Azure DevOps +- ✅ Uses MSBuild + PowerShell for deterministic, version-controlled matrix generation +- ✅ Allows simple opt-in via project properties +- ✅ Maintains backward compatibility with existing non-split tests + +## Current State + +### Existing Split Tests +- **Aspire.Templates.Tests**: Already uses class-based splitting +- Splits into ~10-15 test classes +- Each OS generates its own matrix (separate setup jobs) +- Uses `--filter-class` to run individual classes + +### Problem Statement +3-4 test projects have very long run times: +1. **Aspire.Hosting.Tests** - Very long, needs splitting +2. Likely other Hosting-related tests +3. Some integration test projects + +Currently only Templates.Tests uses splitting. We need a **common mechanism** that: +- Any test project can opt into +- Automatically handles class enumeration +- Generates appropriate matrices for all OSes +- Requires minimal YAML changes + +## Architecture + +### Component Overview + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ GitHub Actions │ +├─────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ │ +│ │ setup_for_ │ │ setup_for_ │ │ setup_for_ │ │ +│ │ tests_lin │ │ tests_macos │ │ tests_win │ │ +│ │ (ubuntu) │ │ (macos) │ │ (windows) │ │ +│ └───────┬───────┘ └───────┬───────┘ └───────┬───────┘ │ +│ │ │ │ │ +│ └─────────┬────────┴────────┬─────────┘ │ +│ ▼ ▼ │ +│ ┌───────────────────────────────────┐ │ +│ │ .github/actions/enumerate-tests │ │ +│ └───────────────┬───────────────────┘ │ +│ ▼ │ +│ ┌───────────────────────────────────┐ │ +│ │ tests/Shared/GetTestProjects.proj │ │ +│ │ (MSBuild orchestration) │ │ +│ └───────────────┬───────────────────┘ │ +│ │ │ +│ ┌──────────────────┼──────────────────┐ │ +│ ▼ ▼ ▼ │ +│ ┌─────────────┐ ┌──────────────┐ ┌───────────────┐ │ +│ │ Regular │ │ Build Split │ │ Generate │ │ +│ │ Tests List │ │ Test Projects│ │ Matrices │ │ +│ └─────────────┘ └──────┬───────┘ └───────┬───────┘ │ +│ │ │ │ +│ ▼ ▼ │ +│ ┌────────────────────────────────────┐ │ +│ │ eng/scripts/generate-test-matrix.ps1│ │ +│ │ (PowerShell - reads .tests.list │ │ +│ │ and .metadata.json files) │ │ +│ └────────────────┬───────────────────┘ │ +│ ▼ │ +│ ┌────────────────────────────────┐ │ +│ │ artifacts/test-matrices/ │ │ +│ │ split-tests-matrix.json │ │ +│ └────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +### Data Flow + +``` +Per-OS Setup Job + ↓ +enumerate-tests action + ↓ +GetTestProjects.proj (MSBuild) + ↓ + ├─→ Regular Tests → .list file + │ + └─→ Split Tests Projects → .list.split-projects + ↓ + Build each split project + ↓ + ExtractTestClassNames target + ↓ + Generate per-project: + ├─→ ProjectName.tests.list (test class names) + └─→ ProjectName.tests.metadata.json (config) + ↓ + generate-test-matrix.ps1 + ↓ + split-tests-matrix.json + ↓ + GitHub Actions matrix +``` + +## Implementation Steps + +See individual files: +1. [Step 1: MSBuild Targets](./STEP_01_MSBUILD_TARGETS.md) +2. [Step 2: PowerShell Script](./STEP_02_POWERSHELL_SCRIPT.md) +3. [Step 3: GitHub Actions](./STEP_03_GITHUB_ACTIONS.md) +4. [Step 4: Project Configuration](./STEP_04_PROJECT_CONFIG.md) +5. [Step 5: Testing & Validation](./STEP_05_TESTING.md) + +## OS-Specific Considerations + +### Per-OS Matrix Generation + +Each OS generates its own matrix in parallel: +- **Linux** (ubuntu-latest): `setup_for_tests_lin` +- **macOS** (macos-latest): `setup_for_tests_macos` +- **Windows** (windows-latest): `setup_for_tests_win` + +This is critical because: +1. Projects can opt-in/out per OS via `RunOnGithubActions{Windows|Linux|MacOS}` properties +2. File paths differ (slash direction) +3. Some tests only run on specific OSes (e.g., Docker on Linux) + +### PowerShell Cross-Platform + +The `generate-test-matrix.ps1` script: +- ✅ Uses PowerShell Core features (cross-platform) +- ✅ Uses `System.IO.Path.Combine()` for path handling +- ✅ Avoids OS-specific cmdlets +- ✅ Tested on all 3 OSes + +## Migration Strategy + +### Phase 1: Infrastructure (Week 1) +- Implement MSBuild targets +- Create PowerShell script +- Update enumerate-tests action +- Test with Aspire.Templates.Tests (already splitting) + +### Phase 2: Enable for Long-Running Tests (Week 2) +- Migrate Aspire.Templates.Tests to new mechanism +- Enable splitting for Aspire.Hosting.Tests +- Enable for 2-3 other long-running projects +- Monitor CI times + +### Phase 3: Optimization (Week 3) +- Analyze job distribution +- Fine-tune timeouts +- Add any missing metadata fields +- Document usage + +## Success Criteria + +- ✅ All OSes generate correct matrices +- ✅ Split tests run in parallel per class +- ✅ Regular tests continue to work unchanged +- ✅ CI time for long-running projects reduced by 50%+ +- ✅ No increase in flakiness +- ✅ Works on both GitHub Actions and Azure DevOps + +## Rollback Plan + +If issues arise: +1. Set `SplitTestsForCI=false` in problematic project +2. Project reverts to regular single-job execution +3. No YAML changes needed (matrix will be empty) + +## Files Modified/Created + +### New Files +- `eng/scripts/generate-test-matrix.ps1` +- `docs/testing/test-splitting.md` (documentation) + +### Modified Files +- `tests/Directory.Build.targets` +- `tests/Shared/GetTestProjects.proj` +- `.github/actions/enumerate-tests/action.yml` +- `.github/workflows/tests.yml` +- `tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj` +- `tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj` (if enabled) + +## Next Steps + +1. Review this plan +2. Begin implementation following step-by-step guides +3. Create PR with Phase 1 changes +4. Test thoroughly on all OSes +5. Gradually roll out to long-running projects + +--- + +**Implementation Details**: See individual step markdown files in this directory. \ No newline at end of file diff --git a/docs/test-splitting/IMPLEMENTATION_PLAN_V2.md b/docs/test-splitting/IMPLEMENTATION_PLAN_V2.md new file mode 100644 index 00000000000..7ee175d2f1a --- /dev/null +++ b/docs/test-splitting/IMPLEMENTATION_PLAN_V2.md @@ -0,0 +1,384 @@ +# Test Splitting Implementation Plan v2 - Hybrid Collection + Class Splitting + +**Date**: 2025-10-16 +**Author**: @radical +**Objective**: Implement a flexible test splitting mechanism that supports: +- ✅ Individual jobs per xUnit Collection (for grouped tests) +- ✅ ONE job for all uncollected tests (catch-all) +- ✅ Works across all 3 OSes (Linux, macOS, Windows) + +## Overview + +This v2 plan enhances the original with **hybrid collection-based splitting**: + +### Splitting Strategies + +``` +Test Project + ├─ Tests with [Collection("Group1")] → 1 job (all Group1 tests) + ├─ Tests with [Collection("Group2")] → 1 job (all Group2 tests) + └─ All other tests (no collection) → 1 job (ClassA + ClassB + ClassC + ...) +``` + +### Example with 3 Jobs + +``` +Aspire.Hosting.Tests + ├─ [Collection("SlowDatabaseTests")] → Job 1: Collection_SlowDatabaseTests + ├─ [Collection("IntegrationTests")] → Job 2: Collection_IntegrationTests + └─ QuickTests, FastTests, UnitTests... → Job 3: UncollectedTests + (no collection attribute) +``` + +**Total**: 3 parallel jobs instead of 1 monolithic job + +### xUnit Collection Features Used + +- `[Collection("name")]` attribute to group test classes +- `--filter-collection ` to run specific collection +- `--filter-not-collection --filter-not-collection ` to run everything NOT in collections + +## Architecture Changes + +### Test Discovery Output Format + +The `.tests.list` file now includes collections discovered: + +``` +# Format: : +collection:SlowDatabaseTests +collection:IntegrationTests +uncollected:* +``` + +Note: We don't list individual classes anymore - just collections + one uncollected entry. + +### Matrix Entry Structure + +```json +{ + "include": [ + { + "type": "collection", + "name": "SlowDatabaseTests", + "filterArg": "--filter-collection SlowDatabaseTests", + "shortname": "Collection_SlowDatabaseTests", + "testSessionTimeout": "30m", + "testHangTimeout": "15m" + }, + { + "type": "collection", + "name": "IntegrationTests", + "filterArg": "--filter-collection IntegrationTests", + "shortname": "Collection_IntegrationTests", + "testSessionTimeout": "25m", + "testHangTimeout": "12m" + }, + { + "type": "uncollected", + "name": "UncollectedTests", + "filterArg": "--filter-not-collection SlowDatabaseTests --filter-not-collection IntegrationTests", + "shortname": "Uncollected", + "testSessionTimeout": "20m", + "testHangTimeout": "10m" + } + ] +} +``` + +## Key Benefits + +### Efficiency +- **Fewer jobs**: Only create jobs for collections + 1 catch-all +- **Less overhead**: No job-per-class overhead for fast tests +- **Better resource usage**: Group related tests with shared fixtures + +### Flexibility +- **Opt-in granularity**: Only split out slow/problematic test groups +- **Simple default**: Tests without collections run normally together +- **Developer control**: Use `[Collection]` to optimize as needed + +### Backward Compatible +- **No collections?** → 1 job (current behavior) +- **All collections?** → N jobs (one per collection) +- **Mixed?** → N+1 jobs (collections + uncollected) + +## Implementation Steps + +See updated files: +1. [Step 1: MSBuild Targets (v2)](./STEP_01_MSBUILD_TARGETS_V2.md) +2. [Step 2: PowerShell Script (v2)](./STEP_02_POWERSHELL_SCRIPT_V2.md) +3. [Step 3: GitHub Actions (No Changes)](./STEP_03_GITHUB_ACTIONS.md) +4. [Step 4: Project Configuration (v2)](./STEP_04_PROJECT_CONFIG_V2.md) +5. [Step 5: Testing & Validation (v2)](./STEP_05_TESTING_V2.md) + +## Usage Examples + +### Example 1: No Collections (Simple Case) + +```xml + + true + Aspire.Hosting.Tests + +``` + +```csharp +// No collection attributes +public class QuickTests { } +public class FastTests { } +public class UnitTests { } +``` + +**Result**: 1 job running all tests (equivalent to not splitting) + +### Example 2: Hybrid Splitting (Recommended) + +```xml + + true + Aspire.Hosting.Tests + +``` + +```csharp +// Slow database tests - group together +[Collection("DatabaseTests")] +public class PostgresTests +{ + // 50 tests, 15 minutes +} + +[Collection("DatabaseTests")] +public class MySqlTests +{ + // 30 tests, 10 minutes +} + +// Slow container tests - separate group +[Collection("ContainerTests")] +public class DockerTests +{ + // 40 tests, 12 minutes +} + +// Fast tests - no collection (run together) +public class QuickTests +{ + // 100 tests, 2 minutes +} + +public class UnitTests +{ + // 200 tests, 3 minutes +} +``` + +**Result**: 3 parallel jobs +1. **Collection_DatabaseTests**: PostgresTests + MySqlTests (~25 min) +2. **Collection_ContainerTests**: DockerTests (~12 min) +3. **UncollectedTests**: QuickTests + UnitTests (~5 min) + +**Total CI time**: ~25 min (previously 55+ min) + +### Example 3: All Collections (Maximum Splitting) + +```csharp +[Collection("PostgresTests")] +public class PostgresTests { } + +[Collection("MySqlTests")] +public class MySqlTests { } + +[Collection("DockerTests")] +public class DockerTests { } +``` + +**Result**: 3 jobs (one per collection), no uncollected job + +### Example 4: Exclude Certain Collections + +```xml + + true + Aspire.Hosting.Tests + + + QuickTests;FastTests + +``` + +```csharp +[Collection("SlowTests")] +public class SlowTests { } // Gets own job + +[Collection("QuickTests")] +public class QuickTests { } // Runs in UncollectedTests job + +public class OtherTests { } // Runs in UncollectedTests job +``` + +**Result**: 2 jobs +1. **Collection_SlowTests** +2. **UncollectedTests** (QuickTests + OtherTests) + +## Configuration Properties + +### New in v2 + +```xml + +false + + +Collection1;Collection2 + + +20m +10m +``` + +### Per-Collection Timeouts (Advanced) + +```xml + + + 30m + 25m + +``` + +## Decision Tree + +``` +Is the test project slow (>15 minutes)? +│ +├─ NO → Don't enable splitting +│ (Keep as regular test) +│ +└─ YES → Do you have groups of slow tests? + │ + ├─ NO → Don't enable splitting OR use simple splitting + │ (All tests in one job is fine) + │ + └─ YES → Use collection-based splitting! + │ + Step 1: Add [Collection("GroupName")] to slow test groups + Step 2: Set SplitTestsForCI=true + Step 3: Set TestClassNamesPrefix + Step 4: Leave fast tests without collection attribute + │ + Result: N+1 jobs (N collections + 1 uncollected) +``` + +## Migration Strategy + +### Phase 1: Infrastructure (Week 1) +- Implement v2 MSBuild targets with collection discovery +- Update PowerShell script to generate collection-based matrices +- Test with example project (no actual collections yet) + +### Phase 2: Migrate Templates.Tests (Week 2) +- Keep NO collections initially (verify 1 job = current behavior) +- Optionally add collections if beneficial +- Validate backward compatibility + +### Phase 3: Enable Hosting.Tests (Week 3) +- Analyze test suite to identify slow groups +- Add `[Collection]` attributes to slow test groups +- Enable `SplitTestsForCI=true` +- Compare CI times before/after + +### Phase 4: Rollout & Optimize (Week 4) +- Apply to other long-running projects +- Fine-tune collection groupings based on actual times +- Document best practices + +## Best Practices + +### When to Use Collections + +✅ **DO** use collections for: +- Tests that share expensive setup/teardown +- Tests that use the same test fixtures +- Long-running integration tests that can be grouped logically +- Tests that have similar resource requirements + +❌ **DON'T** use collections for: +- Fast unit tests (let them run together in uncollected job) +- Tests that should be isolated +- Creating too many tiny collections (overhead not worth it) + +### Recommended Groupings + +```csharp +// Good: Logical grouping of slow tests +[Collection("DatabaseIntegrationTests")] +public class PostgresIntegrationTests { } + +[Collection("DatabaseIntegrationTests")] +public class SqlServerIntegrationTests { } + +// Good: Resource-specific grouping +[Collection("DockerContainerTests")] +public class ContainerLifecycleTests { } + +[Collection("DockerContainerTests")] +public class ContainerNetworkingTests { } + +// Bad: Too granular (defeats the purpose) +[Collection("PostgresTest1")] +public class PostgresTest1 { } + +[Collection("PostgresTest2")] +public class PostgresTest2 { } +``` + +## Expected Outcomes + +### Before (Monolithic) +``` +Aspire.Hosting.Tests: 1 job, 60 minutes +``` + +### After (Collection-Based Splitting) +``` +Collection_DatabaseTests: 1 job, 25 minutes +Collection_ContainerTests: 1 job, 20 minutes +Collection_AzureTests: 1 job, 15 minutes +UncollectedTests: 1 job, 10 minutes +``` + +**Total CI time**: ~25 minutes (jobs run in parallel) +**Job count**: 4 jobs (manageable) +**Time saved**: 35 minutes (58% reduction) + +## Success Criteria + +- ✅ All OSes generate correct collection-based matrices +- ✅ Collection tests run together in single jobs +- ✅ Uncollected tests run together in one job +- ✅ No tests are accidentally skipped +- ✅ CI time for long-running projects reduced by 50%+ +- ✅ Number of jobs remains manageable (<10 per project per OS) +- ✅ Works on both GitHub Actions and Azure DevOps + +## Rollback Plan + +If issues arise: +1. Set `DisableCollectionBasedSplitting=true` to use v1 class-based splitting +2. Or set `SplitTestsForCI=false` to disable all splitting +3. No YAML changes needed (matrix adapts automatically) + +## Next Steps + +1. Review this updated v2 plan +2. Implement Step 1 (MSBuild targets with collection discovery) +3. Implement Step 2 (PowerShell script with collection matrix generation) +4. Test with sample collections +5. Roll out to Hosting.Tests +6. Monitor and optimize + +--- + +**Key Innovation**: v2 uses xUnit collections to create **logical test groups** while keeping fast tests together, resulting in optimal parallelization with minimal job overhead. \ No newline at end of file diff --git a/docs/test-splitting/IMPLEMENTATION_PLAN_V3.md b/docs/test-splitting/IMPLEMENTATION_PLAN_V3.md new file mode 100644 index 00000000000..c22a5662de8 --- /dev/null +++ b/docs/test-splitting/IMPLEMENTATION_PLAN_V3.md @@ -0,0 +1,332 @@ +# Test Splitting Implementation Plan v3 - Auto-Detection + +**Date**: 2025-10-16 +**Author**: @radical +**User**: radical +**Objective**: Implement automatic detection of splitting strategy: +- Collections present → Split by collection + uncollected +- No collections → Split by class (original behavior) +- No `SplitTestsOnCI` → No splitting (run as single job) + +## Overview + +This v3 plan simplifies configuration by automatically detecting the appropriate splitting strategy. + +## Auto-Detection Logic + +``` +Is SplitTestsOnCI=true? + │ + ├─ NO → Run as single job (no splitting) + │ + └─ YES → Build project and extract test metadata + │ + ├─ Has Collections? → Split by Collection + Uncollected + │ Result: N+1 jobs (one per collection + one uncollected) + │ + └─ No Collections? → Split by Class + Result: N jobs (one per test class) +``` + +## Splitting Modes + +### Mode 1: No Splitting (Default) + +```xml + + + +``` + +**Result**: 1 job running entire test project + +### Mode 2: Collection-Based Splitting (Auto-Detected) + +```xml + + true + Aspire.Hosting.Tests + +``` + +```csharp +[Collection("DatabaseTests")] +public class PostgresTests { } + +[Collection("ContainerTests")] +public class DockerTests { } + +public class QuickTests { } // No collection +``` + +**Detection**: Collections found → Use collection-based splitting +**Result**: 3 jobs (DatabaseTests, ContainerTests, Uncollected) + +### Mode 3: Class-Based Splitting (Auto-Detected) + +```xml + + true + Aspire.Templates.Tests + +``` + +```csharp +// No [Collection] attributes on any test class +public class Test1 { } +public class Test2 { } +public class Test3 { } +``` + +**Detection**: No collections found → Use class-based splitting +**Result**: 3 jobs (Test1, Test2, Test3) + +## Architecture + +### Phase 1: Discovery (MSBuild) + +``` +ExtractTestClassNames Target + ↓ +Run: dotnet .dll --list-tests + ↓ +Parse output with PowerShell helper + ↓ +Detect collections using regex + ↓ + ├─ Collections found? + │ └─ Write: collection:Name, uncollected:* + │ + └─ No collections? + └─ Write: class:FullClassName (one per class) +``` + +### Phase 2: Matrix Generation (PowerShell) + +``` +generate-test-matrix.ps1 + ↓ +Read .tests.list file + ↓ +Parse entries + ↓ + ├─ Type: collection + │ └─ Generate: Collection jobs + Uncollected job + │ + └─ Type: class + └─ Generate: One job per class +``` + +## Implementation Components + +### 1. PowerShell Discovery Helper + +New script: `eng/scripts/extract-test-metadata.ps1` + +Parses `--list-tests` output to detect collections. + +### 2. Enhanced MSBuild Target + +`ExtractTestClassNames` target calls PowerShell helper to detect mode. + +### 3. Enhanced Matrix Generator + +`generate-test-matrix.ps1` handles both collection and class entries. + +## File Formats + +### .tests.list Format (Auto-Generated) + +**Collection-based mode** (collections detected): +``` +collection:DatabaseTests +collection:ContainerTests +uncollected:* +``` + +**Class-based mode** (no collections): +``` +class:Aspire.Templates.Tests.Test1 +class:Aspire.Templates.Tests.Test2 +class:Aspire.Templates.Tests.Test3 +``` + +### Matrix Output + +**Collection-based**: +```json +{ + "include": [ + { + "type": "collection", + "name": "DatabaseTests", + "filterArg": "--filter-collection \"DatabaseTests\"", + ... + }, + { + "type": "uncollected", + "name": "UncollectedTests", + "filterArg": "--filter-not-collection \"DatabaseTests\" ...", + ... + } + ] +} +``` + +**Class-based**: +```json +{ + "include": [ + { + "type": "class", + "fullClassName": "Aspire.Templates.Tests.Test1", + "filterArg": "--filter-class \"Aspire.Templates.Tests.Test1\"", + ... + }, + { + "type": "class", + "fullClassName": "Aspire.Templates.Tests.Test2", + "filterArg": "--filter-class \"Aspire.Templates.Tests.Test2\"", + ... + } + ] +} +``` + +## Benefits + +1. **Zero Configuration**: Just set `SplitTestsOnCI=true` and it works +2. **Automatic Optimization**: Uses collections if present, falls back to classes +3. **Backward Compatible**: Existing projects work without changes +4. **Developer-Friendly**: Add `[Collection]` when needed, remove when not +5. **Flexible**: Can mix modes across different projects + +## Configuration Properties + +### Minimal Configuration + +```xml + + + true + YourProject.Tests + +``` + +### Optional Overrides + +```xml + + + 25m + 12m + + + 15m + FastTests + + + false + false + false + +``` + +## Implementation Steps + +1. [Step 1: PowerShell Discovery Helper](./STEP_01_DISCOVERY_HELPER.md) +2. [Step 2: MSBuild Targets (v3)](./STEP_02_MSBUILD_TARGETS_V3.md) +3. [Step 3: Matrix Generator (v3)](./STEP_03_MATRIX_GENERATOR_V3.md) +4. [Step 4: GitHub Actions (No Changes)](./STEP_03_GITHUB_ACTIONS.md) +5. [Step 5: Project Configuration (v3)](./STEP_04_PROJECT_CONFIG_V3.md) +6. [Step 6: Testing & Migration](./STEP_05_TESTING_V3.md) + +## Migration Examples + +### Example 1: Aspire.Templates.Tests + +**Current** (custom mechanism): +```xml +true +Aspire.Templates.Tests +``` + +**After v3** (unified, auto-detect): +```xml +true +Aspire.Templates.Tests +``` + +**Auto-detected mode**: Class-based (no collections in templates tests) +**Result**: Same behavior as before (one job per test class) + +### Example 2: Aspire.Hosting.Tests (NEW) + +```xml +true +Aspire.Hosting.Tests +``` + +**Option A**: Leave tests as-is (no collections) +- **Auto-detected mode**: Class-based +- **Result**: One job per test class (~50 jobs) + +**Option B**: Add collections to slow tests +```csharp +[Collection("DatabaseTests")] +public class PostgresTests { } + +[Collection("DatabaseTests")] +public class MySqlTests { } + +public class QuickTests { } // No collection +``` + +- **Auto-detected mode**: Collection-based +- **Result**: 3 jobs (DatabaseTests, Uncollected with QuickTests, etc.) + +## Decision Tree + +``` +Want to split tests? +│ +├─ NO → Don't set SplitTestsOnCI +│ Result: 1 job (current behavior) +│ +└─ YES → Set SplitTestsOnCI=true + │ + Do you have logical test groups? + │ + ├─ YES → Add [Collection] attributes + │ Result: Auto-detected collection mode + │ Jobs: N collections + 1 uncollected + │ + └─ NO → Leave tests as-is + Result: Auto-detected class mode + Jobs: One per class +``` + +## Success Criteria + +- ✅ Auto-detection works for both modes +- ✅ No breaking changes to existing projects +- ✅ Templates.Tests migrates cleanly +- ✅ Hosting.Tests can use either mode +- ✅ All 3 OSes work correctly +- ✅ Clear logging shows which mode was detected +- ✅ CI times reduced by 50%+ for long-running projects + +## Next Steps + +1. Review v3 plan +2. Implement discovery helper script +3. Update MSBuild targets with auto-detection +4. Update matrix generator to handle both modes +5. Test with both collection and class modes +6. Migrate Templates.Tests as proof-of-concept +7. Enable Hosting.Tests with collections +8. Document best practices + +--- + +**Key Innovation**: v3 uses **automatic detection** to choose the optimal splitting strategy, eliminating configuration complexity while maintaining flexibility. \ No newline at end of file diff --git a/docs/test-splitting/IMPLEMENTATION_SUMMARY.md b/docs/test-splitting/IMPLEMENTATION_SUMMARY.md new file mode 100644 index 00000000000..7b796377399 --- /dev/null +++ b/docs/test-splitting/IMPLEMENTATION_SUMMARY.md @@ -0,0 +1,255 @@ +# Test Splitting Implementation - Summary & Checklist + +**Date**: 2025-01-16 +**Author**: @radical +**Status**: Ready for Implementation + +## Overview + +This implementation adds automatic test splitting to dotnet/aspire CI, reducing test execution time by running tests in parallel. + +**Key Innovation**: Auto-detection of splitting strategy +- Has `[Collection]` attributes? → Split by collection + uncollected +- No collections? → Split by test class +- Not enabled? → Run as single job (no change) + +## What's Being Implemented + +### New Files + +1. **`eng/scripts/extract-test-metadata.ps1`** (Step 1) + - Parses `--list-tests` output + - Detects collections vs classes + - Outputs `.tests.list` file + +2. **`eng/scripts/generate-test-matrix.ps1`** (Step 3) + - Reads `.tests.list` and `.tests.metadata.json` + - Generates JSON matrix for CI + - Handles both collection and class modes + +### Modified Files + +3. **`tests/Directory.Build.targets`** (Step 2) + - Enhanced `ExtractTestClassNames` target + - Calls PowerShell discovery helper + - Writes metadata for matrix generation + +4. **`tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj`** (Step 4) + - Migrate from old custom mechanism + - Use new unified `SplitTestsOnCI` property + +### Existing Files (No Changes) + +- `.github/workflows/tests.yml` - Already supports new matrix format +- `.github/actions/enumerate-tests/action.yml` - Already calls scripts correctly +- `tests/Shared/GetTestProjects.proj` - Already orchestrates correctly + +## Implementation Checklist + +### Phase 1: Infrastructure (Week 1) + +- [ ] **Create `eng/scripts/extract-test-metadata.ps1`** + - [ ] Copy from STEP_01_DISCOVERY_HELPER.md + - [ ] Test with mock data (see Step 5) + - [ ] Verify collections detected correctly + - [ ] Verify class-only mode works + +- [ ] **Create `eng/scripts/generate-test-matrix.ps1`** + - [ ] Copy from STEP_03_MATRIX_GENERATOR_V3.md + - [ ] Test with sample .tests.list files (see Step 5) + - [ ] Verify JSON output is valid + - [ ] Test both collection and class modes + +- [ ] **Update `tests/Directory.Build.targets`** + - [ ] Add enhanced ExtractTestClassNames target from STEP_02_MSBUILD_TARGETS_V3.md + - [ ] Test locally with `dotnet build` (see Step 5) + - [ ] Verify `.tests.list` and `.tests.metadata.json` are created + - [ ] Check binlog for errors + +### Phase 2: Migrate Templates.Tests (Week 2) + +- [ ] **Update `tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj`** + - [ ] Replace `ExtractTestClassNamesForHelix` with `SplitTestsOnCI` + - [ ] Add `RequiresNugetsForSplitTests=true` + - [ ] Add `RequiresTestSdkForSplitTests=true` + - [ ] Add `EnablePlaywrightInstallForSplitTests=true` + - [ ] Remove `TestArchiveTestsDir` override + +- [ ] **Test Locally** + - [ ] Build project with splitting enabled + - [ ] Verify class-based mode detected (no collections in templates tests) + - [ ] Check `.tests.list` has `class:` entries + - [ ] Verify matrix has same number of jobs as before + +- [ ] **Create PR** + - [ ] Title: "Migrate Aspire.Templates.Tests to unified test splitting" + - [ ] Link to this implementation plan + - [ ] Test in CI + - [ ] Verify same behavior as before + +### Phase 3: Enable Hosting.Tests (Week 3) + +- [ ] **Add Collections to Slow Tests** + - [ ] Identify slow test groups (>10 min combined) + - [ ] Add `[Collection("DatabaseTests")]` to database test classes + - [ ] Add `[Collection("ContainerTests")]` to container test classes + - [ ] Leave fast tests without `[Collection]` attribute + +- [ ] **Update `tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj`** + - [ ] Add `SplitTestsOnCI=true` + - [ ] Add `TestClassNamesPrefix=Aspire.Hosting.Tests` + - [ ] Set timeouts (see Step 4) + +- [ ] **Test Locally** + - [ ] Build with splitting enabled + - [ ] Verify collection-based mode detected + - [ ] Check `.tests.list` has `collection:` entries + - [ ] Test filters work (see Step 5) + +- [ ] **Create PR** + - [ ] Title: "Enable test splitting for Aspire.Hosting.Tests" + - [ ] Document expected CI time improvement + - [ ] Monitor CI times after merge + +### Phase 4: Rollout & Optimize (Week 4) + +- [ ] **Identify Other Long-Running Projects** + - [ ] Review CI times for all test projects + - [ ] List projects > 15 minutes + - [ ] Prioritize by impact + +- [ ] **Enable Splitting Incrementally** + - [ ] One project per PR + - [ ] Monitor each for issues + - [ ] Adjust collection groupings as needed + +- [ ] **Document Best Practices** + - [ ] Collection size guidelines + - [ ] When to split vs not split + - [ ] Troubleshooting common issues + +## Testing Strategy + +### Local Testing (Before Each PR) + +1. **Unit Test Scripts** + - [ ] Test `extract-test-metadata.ps1` with mock data + - [ ] Test `generate-test-matrix.ps1` with sample files + - [ ] Verify JSON output structure + +2. **Integration Test MSBuild** + - [ ] Build test project with splitting enabled + - [ ] Verify files generated in `artifacts/helix/` + - [ ] Check mode detection is correct + +3. **End-to-End Test** + - [ ] Run full `GetTestProjects.proj` + - [ ] Generate matrix JSON + - [ ] Validate matrix structure + - [ ] Test xUnit filters work + +### CI Testing (After Push) + +1. **Setup Jobs** + - [ ] All 3 OS setup jobs succeed + - [ ] Matrices are generated + - [ ] Artifacts are uploaded + +2. **Split Test Jobs** + - [ ] New jobs appear as expected + - [ ] Tests run with correct filters + - [ ] Results are uploaded + - [ ] No unexpected failures + +3. **Performance** + - [ ] CI times reduced as expected + - [ ] No increase in flakiness + - [ ] Resource usage acceptable + +## Success Criteria + +### Functional + +- [ ] Auto-detection works (collection vs class mode) +- [ ] Templates.Tests migrates without behavior change +- [ ] Hosting.Tests splits into ~3-5 jobs +- [ ] All tests pass in split jobs +- [ ] Test results are properly reported +- [ ] Works on all 3 OSes (Linux, macOS, Windows) + +### Performance + +- [ ] Hosting.Tests CI time reduced by 50%+ +- [ ] No increase in test flakiness +- [ ] Job count remains manageable (<10 per project per OS) + +### Maintainability + +- [ ] Clear documentation for developers +- [ ] Easy to enable for new projects +- [ ] Easy to troubleshoot issues +- [ ] No breaking changes to existing projects + +## Rollback Plan + +If critical issues arise: + +### Per-Project Rollback + +```xml + + +``` + +Project reverts to single-job execution immediately. + +### Full Rollback + +Revert the PR that modified `Directory.Build.targets`. +All projects revert to original behavior. + +## File Reference + +| Step | File(s) | Purpose | +|------|---------|---------| +| 1 | `STEP_01_DISCOVERY_HELPER.md` | PowerShell script to detect collections/classes | +| 2 | `STEP_02_MSBUILD_TARGETS_V3.md` | MSBuild target that calls discovery helper | +| 3 | `STEP_03_MATRIX_GENERATOR_V3.md` | PowerShell script to generate JSON matrices | +| 4 | `STEP_04_PROJECT_CONFIG_V3.md` | How to configure test projects | +| 5 | `STEP_05_TESTING_V3.md` | Local testing guide | +| 6 | `STEP_06_CI_INTEGRATION.md` | CI verification guide | + +## Questions for Copilot + +Before starting implementation, Copilot should clarify: + +1. **Templates.Tests Migration**: Should we remove the old `enumerate-tests` template-specific logic in the workflow, or keep it as fallback? + +2. **Timeout Defaults**: What should default timeout values be if not specified? + - Suggested: `SplitTestSessionTimeout=20m`, `UncollectedTestsSessionTimeout=15m` + +3. **Collection Naming**: Any conventions or restrictions on collection names? + - Suggested: Alphanumeric + underscore only + +4. **Error Handling**: Should we fail CI if splitting is enabled but no tests found, or fall back to running all tests? + - Suggested: Fail fast to catch configuration errors early + +5. **Artifacts**: Should we always upload `.tests.list` and `.tests.metadata.json` files, even on success? + - Suggested: Yes, for debugging and transparency + +## Ready for Implementation? + +- [x] All design documents complete +- [x] Testing strategy defined +- [x] Success criteria clear +- [x] Rollback plan in place +- [x] Questions for Copilot identified + +**Status**: ✅ Ready to hand off to Copilot for PR creation + +**Estimated Implementation Time**: 2-3 hours for infrastructure + testing + +**Recommended Approach**: Implement in 3 separate PRs: +1. PR #1: Add infrastructure (scripts + targets) - test with Templates.Tests +2. PR #2: Enable Hosting.Tests with collections +3. PR #3: Roll out to remaining long-running projects \ No newline at end of file diff --git a/docs/test-splitting/STEP_01_DISCOVERY_HELPER.md b/docs/test-splitting/STEP_01_DISCOVERY_HELPER.md new file mode 100644 index 00000000000..65b645eb1b9 --- /dev/null +++ b/docs/test-splitting/STEP_01_DISCOVERY_HELPER.md @@ -0,0 +1,368 @@ +# Step 1: PowerShell Discovery Helper + +## Overview + +Create a PowerShell helper script that parses `--list-tests` output to detect xUnit collections and test classes, determining the optimal splitting mode. + +## File: `eng/scripts/extract-test-metadata.ps1` + +### Complete Implementation + +```powershell +<# +.SYNOPSIS + Extracts test metadata (collections or classes) from xUnit test assembly. + +.DESCRIPTION + Parses output of 'dotnet test.dll --list-tests' to determine: + - Are collections present? → Use collection-based splitting + - No collections? → Use class-based splitting + + Outputs a structured list file for consumption by matrix generation. + +.PARAMETER TestAssemblyOutput + The console output from running the test assembly with --list-tests + +.PARAMETER TestClassNamesPrefix + Prefix to filter test classes (e.g., "Aspire.Hosting.Tests") + +.PARAMETER TestCollectionsToSkip + Semicolon-separated list of collection names to exclude from splitting + +.PARAMETER OutputListFile + Path to write the .tests.list file + +.EXAMPLE + $output = & dotnet MyTests.dll --list-tests + .\extract-test-metadata.ps1 -TestAssemblyOutput $output -TestClassNamesPrefix "MyTests" -OutputListFile "./tests.list" + +.NOTES + Author: Aspire Team (@radical) + Date: 2025-10-16 + Version: 3.0 + Requires: PowerShell 7.0+ +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory=$true, HelpMessage="Output from test assembly --list-tests")] + [string[]]$TestAssemblyOutput, + + [Parameter(Mandatory=$true, HelpMessage="Prefix for test class names")] + [string]$TestClassNamesPrefix, + + [Parameter(Mandatory=$false, HelpMessage="Collections to skip (semicolon-separated)")] + [string]$TestCollectionsToSkip = "", + + [Parameter(Mandatory=$true, HelpMessage="Output file path")] + [string]$OutputListFile +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +#region Helper Functions + +function Write-Message { + param( + [string]$Message, + [ValidateSet('Info', 'Success', 'Warning', 'Error', 'Debug')] + [string]$Level = 'Info' + ) + + $prefix = switch ($Level) { + 'Success' { '✅' } + 'Warning' { '⚠️' } + 'Error' { '❌' } + 'Debug' { '🔍' } + default { 'ℹ️' } + } + + Write-Host "$prefix $Message" +} + +#endregion + +#region Parse Test Output + +Write-Message "Parsing test assembly output..." -Level Info + +# xUnit v3 output format when listing tests: +# The test assembly output includes test names with their collection information. +# We need to extract both collections and class names. + +$collections = [System.Collections.Generic.HashSet[string]]::new() +$testClasses = [System.Collections.Generic.HashSet[string]]::new() + +# Regex patterns +$testNameRegex = "^\s*($TestClassNamesPrefix[^\(]+)" +$collectionIndicator = "Collection:" # xUnit prints this before test names in a collection + +$currentCollection = $null + +foreach ($line in $TestAssemblyOutput) { + # Check if this line indicates a collection + if ($line -match "^\s*$collectionIndicator\s*(.+)$") { + $currentCollection = $Matches[1].Trim() + Write-Message " Found collection: $currentCollection" -Level Debug + [void]$collections.Add($currentCollection) + continue + } + + # Check if this is a test name line + if ($line -match $testNameRegex) { + $fullTestName = $Matches[1].Trim() + + # Extract class name from test name + # Format: "Namespace.ClassName.MethodName" + if ($fullTestName -match "^($TestClassNamesPrefix\.[^\.]+)\.") { + $className = $Matches[1] + [void]$testClasses.Add($className) + } + } +} + +#endregion + +#region Filter Collections + +$collectionsToSkipList = if ($TestCollectionsToSkip) { + $TestCollectionsToSkip -split ';' | ForEach-Object { $_.Trim() } | Where-Object { $_ } +} else { + @() +} + +$filteredCollections = $collections | Where-Object { $_ -notin $collectionsToSkipList } + +#endregion + +#region Determine Splitting Mode + +$hasCollections = $filteredCollections.Count -gt 0 +$mode = if ($hasCollections) { "collection" } else { "class" } + +Write-Message "" -Level Info +Write-Message "Detection Results:" -Level Success +Write-Message " Mode: $mode" -Level Info +Write-Message " Collections found: $($collections.Count)" -Level Info +Write-Message " Collections after filtering: $($filteredCollections.Count)" -Level Info +Write-Message " Test classes found: $($testClasses.Count)" -Level Info + +if ($collectionsToSkipList.Count -gt 0) { + Write-Message " Skipped collections: $($collectionsToSkipList -join ', ')" -Level Info +} + +#endregion + +#region Generate Output File + +$outputLines = [System.Collections.Generic.List[string]]::new() + +if ($mode -eq "collection") { + Write-Message "" -Level Info + Write-Message "Using COLLECTION-BASED splitting" -Level Success + + # Add collection entries + foreach ($collection in ($filteredCollections | Sort-Object)) { + $outputLines.Add("collection:$collection") + Write-Message " + Job: Collection_$collection" -Level Debug + } + + # Always add uncollected entry + $outputLines.Add("uncollected:*") + Write-Message " + Job: Uncollected (tests without collections)" -Level Debug + + Write-Message "" -Level Info + Write-Message "Expected jobs: $($filteredCollections.Count + 1) ($($filteredCollections.Count) collections + 1 uncollected)" -Level Success +} +else { + Write-Message "" -Level Info + Write-Message "Using CLASS-BASED splitting" -Level Success + + # Add class entries + foreach ($className in ($testClasses | Sort-Object)) { + $outputLines.Add("class:$className") + $shortName = $className -replace "^$TestClassNamesPrefix\.", "" + Write-Message " + Job: $shortName" -Level Debug + } + + Write-Message "" -Level Info + Write-Message "Expected jobs: $($testClasses.Count) (one per class)" -Level Success +} + +#endregion + +#region Write Output File + +# Ensure output directory exists +$outputDir = [System.IO.Path]::GetDirectoryName($OutputListFile) +if ($outputDir -and -not (Test-Path $outputDir)) { + New-Item -ItemType Directory -Path $outputDir -Force | Out-Null +} + +# Write file +$outputLines | Set-Content -Path $OutputListFile -Encoding UTF8 + +Write-Message "" -Level Info +Write-Message "Output written to: $OutputListFile" -Level Success +Write-Message "Lines: $($outputLines.Count)" -Level Info + +#endregion +``` + +## Usage Examples + +### Example 1: Project with Collections + +```powershell +# Run test assembly +$output = & dotnet artifacts/bin/Aspire.Hosting.Tests/Debug/net9.0/Aspire.Hosting.Tests.dll --list-tests + +# Extract metadata +.\eng\scripts\extract-test-metadata.ps1 ` + -TestAssemblyOutput $output ` + -TestClassNamesPrefix "Aspire.Hosting.Tests" ` + -OutputListFile "./artifacts/helix/Aspire.Hosting.Tests.tests.list" +``` + +**Console Output**: +``` +ℹ️ Parsing test assembly output... +🔍 Found collection: DatabaseTests +🔍 Found collection: ContainerTests + +✅ Detection Results: +ℹ️ Mode: collection +ℹ️ Collections found: 2 +ℹ️ Collections after filtering: 2 +ℹ️ Test classes found: 15 + +✅ Using COLLECTION-BASED splitting +🔍 + Job: Collection_DatabaseTests +🔍 + Job: Collection_ContainerTests +🔍 + Job: Uncollected (tests without collections) + +✅ Expected jobs: 3 (2 collections + 1 uncollected) + +✅ Output written to: ./artifacts/helix/Aspire.Hosting.Tests.tests.list +ℹ️ Lines: 3 +``` + +**Output File** (`Aspire.Hosting.Tests.tests.list`): +``` +collection:ContainerTests +collection:DatabaseTests +uncollected:* +``` + +### Example 2: Project without Collections + +```powershell +$output = & dotnet artifacts/bin/Aspire.Templates.Tests/Debug/net9.0/Aspire.Templates.Tests.dll --list-tests + +.\eng\scripts\extract-test-metadata.ps1 ` + -TestAssemblyOutput $output ` + -TestClassNamesPrefix "Aspire.Templates.Tests" ` + -OutputListFile "./artifacts/helix/Aspire.Templates.Tests.tests.list" +``` + +**Console Output**: +``` +ℹ️ Parsing test assembly output... + +✅ Detection Results: +ℹ️ Mode: class +ℹ️ Collections found: 0 +ℹ️ Collections after filtering: 0 +ℹ️ Test classes found: 12 + +✅ Using CLASS-BASED splitting +🔍 + Job: BuildAndRunStarterTemplateBuiltInTest +🔍 + Job: BuildAndRunTemplateTests +🔍 + Job: EmptyTemplateRunTests +... + +✅ Expected jobs: 12 (one per class) + +✅ Output written to: ./artifacts/helix/Aspire.Templates.Tests.tests.list +ℹ️ Lines: 12 +``` + +**Output File** (`Aspire.Templates.Tests.tests.list`): +``` +class:Aspire.Templates.Tests.BuildAndRunStarterTemplateBuiltInTest +class:Aspire.Templates.Tests.BuildAndRunTemplateTests +class:Aspire.Templates.Tests.EmptyTemplateRunTests +class:Aspire.Templates.Tests.MSTest_PerTestFrameworkTemplatesTests +class:Aspire.Templates.Tests.NewUpAndBuildStandaloneTemplateTests +class:Aspire.Templates.Tests.None_StarterTemplateProjectNamesTests +class:Aspire.Templates.Tests.Nunit_PerTestFrameworkTemplatesTests +class:Aspire.Templates.Tests.Nunit_StarterTemplateProjectNamesTests +class:Aspire.Templates.Tests.StarterTemplateRunTests +class:Aspire.Templates.Tests.StarterTemplateWithTestsRunTests +class:Aspire.Templates.Tests.Xunit_PerTestFrameworkTemplatesTests +class:Aspire.Templates.Tests.Xunit_StarterTemplateProjectNamesTests +``` + +### Example 3: Skip Certain Collections + +```powershell +.\eng\scripts\extract-test-metadata.ps1 ` + -TestAssemblyOutput $output ` + -TestClassNamesPrefix "Aspire.Hosting.Tests" ` + -TestCollectionsToSkip "QuickTests;FastTests" ` + -OutputListFile "./artifacts/helix/Aspire.Hosting.Tests.tests.list" +``` + +**Result**: QuickTests and FastTests won't get their own jobs; they'll run in the uncollected job. + +## Testing the Script + +### Test 1: Mock Collection Output + +```powershell +$mockOutput = @( + "Collection: DatabaseTests", + " Aspire.Hosting.Tests.PostgresTests.CanStartContainer", + " Aspire.Hosting.Tests.PostgresTests.CanConnectToDatabase", + "Collection: ContainerTests", + " Aspire.Hosting.Tests.DockerTests.CanStartGenericContainer", + "Aspire.Hosting.Tests.QuickTests.FastTest1", + "Aspire.Hosting.Tests.QuickTests.FastTest2" +) + +.\eng\scripts\extract-test-metadata.ps1 ` + -TestAssemblyOutput $mockOutput ` + -TestClassNamesPrefix "Aspire.Hosting.Tests" ` + -OutputListFile "./test-output.list" +``` + +**Expected**: +- Mode: collection +- Collections: DatabaseTests, ContainerTests +- Output: 3 lines (2 collections + uncollected) + +### Test 2: Mock Class-Only Output + +```powershell +$mockOutput = @( + "Aspire.Templates.Tests.Test1.Method1", + "Aspire.Templates.Tests.Test1.Method2", + "Aspire.Templates.Tests.Test2.Method1", + "Aspire.Templates.Tests.Test3.Method1" +) + +.\eng\scripts\extract-test-metadata.ps1 ` + -TestAssemblyOutput $mockOutput ` + -TestClassNamesPrefix "Aspire.Templates.Tests" ` + -OutputListFile "./test-output.list" +``` + +**Expected**: +- Mode: class +- Classes: Test1, Test2, Test3 +- Output: 3 lines (one per class) + +## Next Steps + +Proceed to [Step 2: MSBuild Targets (v3)](./STEP_02_MSBUILD_TARGETS_V3.md) \ No newline at end of file diff --git a/docs/test-splitting/STEP_01_MSBUILD_TARGETS.md b/docs/test-splitting/STEP_01_MSBUILD_TARGETS.md new file mode 100644 index 00000000000..8ceb86d41b1 --- /dev/null +++ b/docs/test-splitting/STEP_01_MSBUILD_TARGETS.md @@ -0,0 +1,236 @@ +# Step 1: MSBuild Targets Implementation + +## Overview + +Modify MSBuild targets to support unified test splitting mechanism while maintaining all 3 OS compatibility. + +## File: `tests/Directory.Build.targets` + +### Changes Required + +1. **Add new ExtractTestClassNames target** (replacing existing) +2. **Add metadata generation** +3. **Update GetRunTestsOnGithubActions target** + +### Implementation + +```xml + + + + + + + + + + + + + + + + + + + + + + <_Regex>^\s*($(TestClassNamesPrefix)[^\($]+) + + + + + <_TestLines0 Include="$([System.Text.RegularExpressions.Regex]::Match('%(_ListOfTestsLines.Identity)', '$(_Regex)'))" /> + + + + + + + + + + + + + + + + + <_RelativeProjectPath>$(MSBuildProjectDirectory.Replace('$(RepoRoot)', '')) + <_RelativeProjectPath>$(_RelativeProjectPath.Replace('\', '/')) + + + + <_MetadataLines Include="{" /> + <_MetadataLines Include=" "projectName": "$(MSBuildProjectName)"," /> + <_MetadataLines Include=" "testClassNamesPrefix": "$(TestClassNamesPrefix)"," /> + <_MetadataLines Include=" "testProjectPath": "$(_RelativeProjectPath)/$(MSBuildProjectFile)"," /> + <_MetadataLines Include=" "requiresNugets": "$(RequiresNugetsForSplitTests)"," /> + <_MetadataLines Include=" "requiresTestSdk": "$(RequiresTestSdkForSplitTests)"," /> + <_MetadataLines Include=" "testSessionTimeout": "$(SplitTestSessionTimeout)"," /> + <_MetadataLines Include=" "testHangTimeout": "$(SplitTestHangTimeout)"," /> + <_MetadataLines Include=" "enablePlaywrightInstall": "$(EnablePlaywrightInstallForSplitTests)"" /> + <_MetadataLines Include="}" /> + + + + + + + + + + + + + + + + + + + + +``` + +## File: `tests/Shared/GetTestProjects.proj` + +### Complete Replacement + +```xml + + + + + $(MSBuildThisFileDirectory)..\..\ + $(ArtifactsDir)test-matrices\ + + + + + + + + + + + + <_TestProjectsToExclude Include="$(RepoRoot)tests\Shared\**\*Tests.csproj" /> + <_TestProjectsToExclude Include="$(RepoRoot)tests\testproject\**\*Tests.csproj" /> + <_TestProjectsToExclude Include="$(RepoRoot)tests\TestingAppHost1\**\*Tests.csproj" /> + + + <_TestProjectsToExclude Include="$(RepoRoot)tests\Aspire.EndToEnd.Tests\**\*Tests.csproj" /> + + <_TestProjects Include="$(RepoRoot)tests\**\*Tests.csproj" + Exclude="@(_TestProjectsToExclude)" /> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <_GenerateMatrixScript>$(RepoRoot)eng\scripts\generate-test-matrix.ps1 + <_TestListsDir>$(ArtifactsDir)helix\ + + + + + + + + +``` + +## Testing the MSBuild Changes + +### Local Testing + +```bash +# On Linux/macOS +./build.sh -restore -build -projects tests/Shared/GetTestProjects.proj /p:TestsListOutputPath=$PWD/artifacts/test-list.txt /p:ContinuousIntegrationBuild=true + +# On Windows +.\build.cmd -restore -build -projects tests/Shared/GetTestProjects.proj /p:TestsListOutputPath=%CD%\artifacts\test-list.txt /p:ContinuousIntegrationBuild=true +``` + +### Verify Outputs + +Check these files were created: +- `artifacts/TestsForGithubActions.list` - Regular tests +- `artifacts/TestsForGithubActions.list.split-projects` - Projects to split (if any) + +### Common Issues + +1. **Path separators**: Ensure paths use `/` in JSON output +2. **Empty lists**: If no split projects, `.split-projects` file won't exist (this is OK) +3. **BuildOs detection**: Make sure `BuildOs` property is set correctly + +## Next Steps + +Proceed to [Step 2: PowerShell Script](./STEP_02_POWERSHELL_SCRIPT.md) \ No newline at end of file diff --git a/docs/test-splitting/STEP_01_MSBUILD_TARGETS_V2.md b/docs/test-splitting/STEP_01_MSBUILD_TARGETS_V2.md new file mode 100644 index 00000000000..23f0637edf9 --- /dev/null +++ b/docs/test-splitting/STEP_01_MSBUILD_TARGETS_V2.md @@ -0,0 +1,295 @@ +# Step 1: MSBuild Targets Implementation (v2 - Collection Support) + +## Overview + +Enhanced MSBuild targets that discover xUnit collections and generate a hybrid matrix with: +- One job per collection +- One job for all uncollected tests + +## File: `tests/Directory.Build.targets` + +### Complete Enhanced Target + +```xml + + + + + + + + + + + + $(RepoRoot)tests\helix\xunit.runner.json + $(RepositoryEngineeringDir)testing\xunit.runner.json + + + $(TestingPlatformCommandLineArguments) --filter-method $(TestMethod) + $(TestingPlatformCommandLineArguments) --filter-class $(TestClass) + $(TestingPlatformCommandLineArguments) --filter-namespace $(TestNamespace) + + true + false + + + false + + + + + + + + + + + $(OutDir) + + -$(TargetFramework) + + + + + + + + + + + + + + + + + + + + <_CollectionRegex>^\s*Collection:\s*(.+)$ + + + <_ClassRegex>^\s*($(TestClassNamesPrefix)[^\($]+) + + + + + <_CollectionLines Include="$([System.Text.RegularExpressions.Regex]::Match('%(_ListOfTestsLinesWithTraits.Identity)', '$(_CollectionRegex)'))" /> + <_CollectionNames Include="$([System.Text.RegularExpressions.Regex]::Match('%(_CollectionLines.Identity)', '$(_CollectionRegex)').Groups[1].Value)" + Condition="'$([System.Text.RegularExpressions.Regex]::Match('%(_CollectionLines.Identity)', '$(_CollectionRegex)').Success)' == 'true'" /> + + + + + + + + + <_HasCollections>false + <_HasCollections Condition="'@(UniqueCollections->Count())' != '0'">true + + + + + + <_TestListLines Include="collection:%(UniqueCollections.Identity)" /> + + + <_TestListLines Include="uncollected:*" /> + + + + + + + + <_RelativeProjectPath>$(MSBuildProjectDirectory.Replace('$(RepoRoot)', '')) + <_RelativeProjectPath>$(_RelativeProjectPath.Replace('\', '/')) + + + <_CollectionsList>@(UniqueCollections, ';') + + + + <_MetadataLines Include="{" /> + <_MetadataLines Include=" "projectName": "$(MSBuildProjectName)"," /> + <_MetadataLines Include=" "testClassNamesPrefix": "$(TestClassNamesPrefix)"," /> + <_MetadataLines Include=" "testProjectPath": "$(_RelativeProjectPath)/$(MSBuildProjectFile)"," /> + <_MetadataLines Include=" "collections": "$(_CollectionsList)"," /> + <_MetadataLines Include=" "requiresNugets": "$(RequiresNugetsForSplitTests)"," /> + <_MetadataLines Include=" "requiresTestSdk": "$(RequiresTestSdkForSplitTests)"," /> + <_MetadataLines Include=" "testSessionTimeout": "$(SplitTestSessionTimeout)"," /> + <_MetadataLines Include=" "testHangTimeout": "$(SplitTestHangTimeout)"," /> + <_MetadataLines Include=" "uncollectedTestsSessionTimeout": "$(UncollectedTestsSessionTimeout)"," /> + <_MetadataLines Include=" "uncollectedTestsHangTimeout": "$(UncollectedTestsHangTimeout)"," /> + <_MetadataLines Include=" "enablePlaywrightInstall": "$(EnablePlaywrightInstallForSplitTests)"" /> + <_MetadataLines Include="}" /> + + + + + + + + + + + + + + + + + + + + + +``` + +## Key Changes from v1 + +### 1. Collection Discovery + +```xml + + + + +``` + +### 2. Collection Extraction + +```xml + + + + +``` + +### 3. Simplified Test List Format + +```xml + + + <_TestListLines Include="collection:%(UniqueCollections.Identity)" /> + <_TestListLines Include="uncollected:*" /> + +``` + +### 4. Collection Metadata + +```xml + +<_MetadataLines Include=" "collections": "$(_CollectionsList)"," /> +``` + +## Testing the MSBuild Changes + +### Test 1: Project with No Collections + +```bash +# Create a test project without collections +dotnet build tests/SomeProject.Tests/SomeProject.Tests.csproj \ + /t:Build;ExtractTestClassNames \ + -p:PrepareForHelix=true \ + -p:SplitTestsForCI=true \ + -p:TestClassNamesPrefix=SomeProject.Tests +``` + +**Expected `.tests.list` output**: +``` +uncollected:* +``` + +**Expected matrix**: 1 job (UncollectedTests) + +### Test 2: Project with Collections + +Add collections to test classes: + +```csharp +[Collection("DatabaseTests")] +public class PostgresTests { } + +[Collection("DatabaseTests")] +public class MySqlTests { } + +[Collection("ContainerTests")] +public class DockerTests { } + +public class QuickTests { } // No collection +``` + +Build: +```bash +dotnet build tests/SomeProject.Tests/SomeProject.Tests.csproj \ + /t:Build;ExtractTestClassNames \ + -p:PrepareForHelix=true \ + -p:SplitTestsForCI=true \ + -p:TestClassNamesPrefix=SomeProject.Tests +``` + +**Expected `.tests.list` output**: +``` +collection:DatabaseTests +collection:ContainerTests +uncollected:* +``` + +**Expected matrix**: 3 jobs +1. Collection_DatabaseTests +2. Collection_ContainerTests +3. UncollectedTests + +### Test 3: Exclude Collections + +```bash +dotnet build tests/SomeProject.Tests/SomeProject.Tests.csproj \ + /t:Build;ExtractTestClassNames \ + -p:PrepareForHelix=true \ + -p:SplitTestsForCI=true \ + -p:TestClassNamesPrefix=SomeProject.Tests \ + -p:TestCollectionsToSkipSplitting=DatabaseTests +``` + +**Expected `.tests.list` output**: +``` +collection:ContainerTests +uncollected:* +``` + +**Expected matrix**: 2 jobs +1. Collection_ContainerTests +2. UncollectedTests (includes DatabaseTests now) + +## File: `tests/Shared/GetTestProjects.proj` + +No changes needed from v1 - this file just orchestrates the builds and calls the PowerShell script. + +## Next Steps + +Proceed to [Step 2: PowerShell Script (v2)](./STEP_02_POWERSHELL_SCRIPT_V2.md) \ No newline at end of file diff --git a/docs/test-splitting/STEP_02_MSBUILD_TARGETS_V3.md b/docs/test-splitting/STEP_02_MSBUILD_TARGETS_V3.md new file mode 100644 index 00000000000..3b2fc761ac6 --- /dev/null +++ b/docs/test-splitting/STEP_02_MSBUILD_TARGETS_V3.md @@ -0,0 +1,446 @@ +# Step 2: MSBuild Targets Implementation (v3 - Auto-Detection) + +## Overview + +Enhanced MSBuild targets that use the PowerShell discovery helper to automatically detect whether to use collection-based or class-based splitting. + +## File: `tests/Directory.Build.targets` + +### Complete Enhanced Implementation + +```xml + + + + + + + + + $(RepoRoot)tests\helix\xunit.runner.json + $(RepositoryEngineeringDir)testing\xunit.runner.json + + + $(TestingPlatformCommandLineArguments) --filter-method $(TestMethod) + $(TestingPlatformCommandLineArguments) --filter-class $(TestClass) + $(TestingPlatformCommandLineArguments) --filter-namespace $(TestNamespace) + + true + + false + + + + + + + + + + + $(OutDir) + + -$(TargetFramework) + + + + + + + + + + + + + + + + + + + + + <_DiscoveryScriptPath>$(RepoRoot)eng\scripts\extract-test-metadata.ps1 + + + <_TestListFile>$(TestArchiveTestsDir)$(MSBuildProjectName).tests.list + <_MetadataFile>$(TestArchiveTestsDir)$(MSBuildProjectName).tests.metadata.json + + + <_RelativeProjectPath>$(MSBuildProjectDirectory.Replace('$(RepoRoot)', '')) + <_RelativeProjectPath>$(_RelativeProjectPath.Replace('\', '/')) + + + <_CollectionsToSkip Condition="'$(TestCollectionsToSkipSplitting)' != ''">$(TestCollectionsToSkipSplitting) + <_CollectionsToSkip Condition="'$(TestCollectionsToSkipSplitting)' == ''"> + + + + + + + + <_TempOutputFile>$(TestArchiveTestsDir)$(MSBuildProjectName).tests.output.tmp + + + + + + + + + <_DiscoveryCommand>pwsh -NoProfile -ExecutionPolicy Bypass -File "$(_DiscoveryScriptPath)" + <_DiscoveryCommand>$(_DiscoveryCommand) -TestAssemblyOutput (Get-Content '$(_TempOutputFile)') + <_DiscoveryCommand>$(_DiscoveryCommand) -TestClassNamesPrefix "$(TestClassNamesPrefix)" + <_DiscoveryCommand Condition="'$(_CollectionsToSkip)' != ''">$(_DiscoveryCommand) -TestCollectionsToSkip "$(_CollectionsToSkip)" + <_DiscoveryCommand>$(_DiscoveryCommand) -OutputListFile "$(_TestListFile)" + + + + + + + + + + + + + + + + + + <_FirstLine>@(_GeneratedListLines->WithMetadataValue('Identity', '@(_GeneratedListLines, 0)')) + <_DetectedMode Condition="$(_FirstLine.StartsWith('collection:'))">collection + <_DetectedMode Condition="$(_FirstLine.StartsWith('class:'))">class + + + <_EntryCount>@(_GeneratedListLines->Count()) + + + + + + + + <_CollectionsList> + + + + <_CollectionLines Include="@(_GeneratedListLines)" Condition="$([System.String]::Copy('%(Identity)').StartsWith('collection:'))" /> + <_CollectionNames Include="$([System.String]::Copy('%(_CollectionLines.Identity)').Substring(11))" /> + + + + <_CollectionsList>@(_CollectionNames, ';') + + + + + <_MetadataLines Include="{" /> + <_MetadataLines Include=" "projectName": "$(MSBuildProjectName)"," /> + <_MetadataLines Include=" "testClassNamesPrefix": "$(TestClassNamesPrefix)"," /> + <_MetadataLines Include=" "testProjectPath": "$(_RelativeProjectPath)/$(MSBuildProjectFile)"," /> + <_MetadataLines Include=" "mode": "$(_DetectedMode)"," /> + <_MetadataLines Include=" "collections": "$(_CollectionsList)"," /> + <_MetadataLines Include=" "requiresNugets": "$(RequiresNugetsForSplitTests)"," /> + <_MetadataLines Include=" "requiresTestSdk": "$(RequiresTestSdkForSplitTests)"," /> + <_MetadataLines Include=" "testSessionTimeout": "$(SplitTestSessionTimeout)"," /> + <_MetadataLines Include=" "testHangTimeout": "$(SplitTestHangTimeout)"," /> + <_MetadataLines Include=" "uncollectedTestsSessionTimeout": "$(UncollectedTestsSessionTimeout)"," /> + <_MetadataLines Include=" "uncollectedTestsHangTimeout": "$(UncollectedTestsHangTimeout)"," /> + <_MetadataLines Include=" "enablePlaywrightInstall": "$(EnablePlaywrightInstallForSplitTests)"" /> + <_MetadataLines Include="}" /> + + + + + + + + + + + + + + + + + + + + + + +``` + +## Key Features + +### 1. PowerShell Helper Integration + +```xml + + +``` + +The command passes: +- Test assembly output (--list-tests results) +- Test class prefix for filtering +- Collections to skip (optional) +- Output file path + +### 2. Automatic Mode Detection + +```xml + + + <_DetectedMode Condition="$(_FirstLine.StartsWith('collection:'))">collection + <_DetectedMode Condition="$(_FirstLine.StartsWith('class:'))">class + +``` + +### 3. Metadata Generation + +The metadata file includes the detected mode: + +```json +{ + "mode": "collection", // or "class" + "collections": "DatabaseTests;ContainerTests", + ... +} +``` + +## Testing the MSBuild Target + +### Test 1: Project with Collections + +Create a test project with collections: + +```csharp +// Aspire.Hosting.Tests/DatabaseTests.cs +[Collection("DatabaseTests")] +public class PostgresTests { } + +[Collection("DatabaseTests")] +public class MySqlTests { } + +// Aspire.Hosting.Tests/QuickTests.cs +public class QuickTests { } // No collection +``` + +Build: + +```bash +dotnet build tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj \ + /t:Build;ExtractTestClassNames \ + -p:PrepareForHelix=true \ + -p:SplitTestsOnCI=true \ + -p:TestClassNamesPrefix=Aspire.Hosting.Tests \ + /bl:test.binlog +``` + +**Expected Console Output**: +``` +[Aspire.Hosting.Tests] Starting test metadata extraction... +[Aspire.Hosting.Tests] Running discovery helper... +ℹ️ Parsing test assembly output... +🔍 Found collection: DatabaseTests +✅ Detection Results: +ℹ️ Mode: collection +ℹ️ Collections found: 1 +... +[Aspire.Hosting.Tests] Detected mode: collection +[Aspire.Hosting.Tests] Generated entries: 2 +[Aspire.Hosting.Tests] ✅ Test metadata extraction complete! +``` + +**Check Output Files**: + +```bash +# List file +cat artifacts/helix/Aspire.Hosting.Tests.tests.list +# collection:DatabaseTests +# uncollected:* + +# Metadata file +cat artifacts/helix/Aspire.Hosting.Tests.tests.metadata.json | jq .mode +# "collection" +``` + +### Test 2: Project without Collections + +```csharp +// Aspire.Templates.Tests/Test1.cs +public class Test1 { } + +// Aspire.Templates.Tests/Test2.cs +public class Test2 { } +``` + +Build: + +```bash +dotnet build tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj \ + /t:Build;ExtractTestClassNames \ + -p:PrepareForHelix=true \ + -p:SplitTestsOnCI=true \ + -p:TestClassNamesPrefix=Aspire.Templates.Tests +``` + +**Expected Console Output**: +``` +[Aspire.Templates.Tests] Starting test metadata extraction... +[Aspire.Templates.Tests] Running discovery helper... +ℹ️ Parsing test assembly output... +✅ Detection Results: +ℹ️ Mode: class +ℹ️ Test classes found: 12 +... +[Aspire.Templates.Tests] Detected mode: class +[Aspire.Templates.Tests] Generated entries: 12 +[Aspire.Templates.Tests] ✅ Test metadata extraction complete! +``` + +**Check Output Files**: + +```bash +# List file +cat artifacts/helix/Aspire.Templates.Tests.tests.list +# class:Aspire.Templates.Tests.Test1 +# class:Aspire.Templates.Tests.Test2 +# ... + +# Metadata file +cat artifacts/helix/Aspire.Templates.Tests.tests.metadata.json | jq .mode +# "class" +``` + +### Test 3: With Skipped Collections + +```bash +dotnet build tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj \ + /t:Build;ExtractTestClassNames \ + -p:PrepareForHelix=true \ + -p:SplitTestsOnCI=true \ + -p:TestClassNamesPrefix=Aspire.Hosting.Tests \ + -p:TestCollectionsToSkipSplitting="QuickTests;FastTests" +``` + +**Result**: QuickTests and FastTests won't appear in collection list; they'll run in uncollected job. + +## Debugging + +### View Binlog + +```bash +# Install dotnet-binlog if not already installed +dotnet tool install -g dotnet-binlog + +# View the binlog +dotnet-binlog test.binlog +``` + +Look for: +- ExtractTestClassNames target execution +- Console output from test assembly +- PowerShell script execution +- Generated file contents + +### Common Issues + +#### Issue 1: "Discovery helper failed" + +**Symptom**: Target fails with error about missing output file +**Cause**: PowerShell script errored +**Fix**: Check script output in binlog; may need to update regex patterns + +#### Issue 2: "No tests found" + +**Symptom**: Empty .tests.list file +**Cause**: TestClassNamesPrefix doesn't match test namespace +**Fix**: Verify prefix matches actual test namespace + +#### Issue 3: "Mode is empty" + +**Symptom**: `$(_DetectedMode)` is blank +**Cause**: Generated file has unexpected format +**Fix**: Check .tests.list file content manually + +### Manual Verification + +```bash +# Check generated files +ls -la artifacts/helix/*.tests.list +ls -la artifacts/helix/*.tests.metadata.json + +# View contents +cat artifacts/helix/YourProject.Tests.tests.list +cat artifacts/helix/YourProject.Tests.tests.metadata.json | jq . + +# Verify mode detection +cat artifacts/helix/YourProject.Tests.tests.metadata.json | jq -r .mode +# Should output: "collection" or "class" +``` + +## File: `tests/Shared/GetTestProjects.proj` + +### No Changes Needed + +The existing v1 implementation works fine - it just calls MSBuild targets and then the PowerShell matrix generator. + +```xml + + + $(MSBuildThisFileDirectory)..\..\ + $(ArtifactsDir)test-matrices\ + + + + + + + + + + + + + + + +``` + +## Next Steps + +Proceed to [Step 3: Matrix Generator (v3)](./STEP_03_MATRIX_GENERATOR_V3.md) \ No newline at end of file diff --git a/docs/test-splitting/STEP_02_POWERSHELL_SCRIPT.md b/docs/test-splitting/STEP_02_POWERSHELL_SCRIPT.md new file mode 100644 index 00000000000..c817c46f40d --- /dev/null +++ b/docs/test-splitting/STEP_02_POWERSHELL_SCRIPT.md @@ -0,0 +1,374 @@ +# Step 2: PowerShell Matrix Generation Script + +## Overview + +Create a cross-platform PowerShell script that reads test class lists and generates JSON matrices for CI consumption. + +## File: `eng/scripts/generate-test-matrix.ps1` + +### Complete Implementation + +```powershell +<# +.SYNOPSIS + Generates CI test matrices from test class enumeration files. + +.DESCRIPTION + This script reads .tests.list and .tests.metadata.json files produced by the + ExtractTestClassNames MSBuild target and generates a JSON matrix file for + consumption by GitHub Actions or Azure DevOps. + + The script is cross-platform and runs on Windows, Linux, and macOS. + +.PARAMETER TestListsDirectory + Directory containing .tests.list and .tests.metadata.json files. + Typically: artifacts/helix/ + +.PARAMETER OutputDirectory + Directory where the JSON matrix file will be written. + Typically: artifacts/test-matrices/ + +.PARAMETER BuildOs + Current operating system being built for (windows, linux, darwin). + Used for logging and debugging. + +.EXAMPLE + pwsh generate-test-matrix.ps1 -TestListsDirectory ./artifacts/helix -OutputDirectory ./artifacts/matrices -BuildOs linux + +.NOTES + Author: Aspire Team + Date: 2025-10-16 + Requires: PowerShell 7.0+ (cross-platform) +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory=$true, HelpMessage="Directory containing test list files")] + [ValidateScript({Test-Path $_ -PathType Container})] + [string]$TestListsDirectory, + + [Parameter(Mandatory=$true, HelpMessage="Output directory for matrix JSON")] + [string]$OutputDirectory, + + [Parameter(Mandatory=$false, HelpMessage="Current OS: windows, linux, or darwin")] + [ValidateSet('windows', 'linux', 'darwin', '')] + [string]$BuildOs = '' +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +#region Helper Functions + +function Write-Message { + <# + .SYNOPSIS + Writes a formatted message to the console. + #> + param( + [Parameter(Mandatory=$true)] + [AllowEmptyString()] + [string]$Message, + + [Parameter(Mandatory=$false)] + [ValidateSet('Info', 'Success', 'Warning', 'Error')] + [string]$Level = 'Info' + ) + + $prefix = switch ($Level) { + 'Success' { '✅' } + 'Warning' { '⚠️' } + 'Error' { '❌' } + default { 'ℹ️' } + } + + $color = switch ($Level) { + 'Success' { 'Green' } + 'Warning' { 'Yellow' } + 'Error' { 'Red' } + default { 'Cyan' } + } + + Write-Host "$prefix $Message" -ForegroundColor $color +} + +function Get-TestListFiles { + <# + .SYNOPSIS + Finds all .tests.list files in the specified directory. + #> + param([string]$Directory) + + Get-ChildItem -Path $Directory -Filter "*.tests.list" -Recurse -ErrorAction SilentlyContinue +} + +function Read-TestMetadata { + <# + .SYNOPSIS + Reads and parses test metadata JSON file. + #> + param( + [string]$MetadataFile, + [string]$ProjectName + ) + + # Default metadata values + $defaults = @{ + testClassNamesPrefix = $ProjectName + testProjectPath = "tests/$ProjectName/$ProjectName.csproj" + requiresNugets = 'false' + requiresTestSdk = 'false' + testSessionTimeout = '20m' + testHangTimeout = '10m' + enablePlaywrightInstall = 'false' + } + + if (-not (Test-Path $MetadataFile)) { + Write-Message "No metadata file found for $ProjectName, using defaults" -Level Warning + return $defaults + } + + try { + $content = Get-Content $MetadataFile -Raw | ConvertFrom-Json + + # Merge with defaults (content overrides defaults) + foreach ($key in $content.PSObject.Properties.Name) { + $defaults[$key] = $content.$key + } + + return $defaults + } + catch { + Write-Message "Failed to parse metadata for ${ProjectName}: $_" -Level Warning + return $defaults + } +} + +function New-MatrixEntry { + <# + .SYNOPSIS + Creates a matrix entry object for a test class. + #> + param( + [string]$FullClassName, + [string]$ProjectName, + [hashtable]$Metadata + ) + + $prefix = $Metadata.testClassNamesPrefix + $shortname = $FullClassName + + # Strip prefix if present (e.g., "Aspire.Templates.Tests.MyClass" → "MyClass") + if ($prefix -and $FullClassName.StartsWith("$prefix.")) { + $shortname = $FullClassName.Substring($prefix.Length + 1) + } + + # Return ordered hashtable for consistent JSON output + [ordered]@{ + shortname = $shortname + projectName = $ProjectName + fullClassName = $FullClassName + testProjectPath = $Metadata.testProjectPath + requiresNugets = ($Metadata.requiresNugets -eq 'true') + requiresTestSdk = ($Metadata.requiresTestSdk -eq 'true') + testSessionTimeout = $Metadata.testSessionTimeout + testHangTimeout = $Metadata.testHangTimeout + enablePlaywrightInstall = ($Metadata.enablePlaywrightInstall -eq 'true') + } +} + +#endregion + +#region Main Script + +Write-Message "Starting matrix generation for BuildOs=$BuildOs" +Write-Message "Test lists directory: $TestListsDirectory" +Write-Message "Output directory: $OutputDirectory" + +# Find all test list files +$listFiles = Get-TestListFiles -Directory $TestListsDirectory + +if ($listFiles.Count -eq 0) { + Write-Message "No test list files found in $TestListsDirectory" -Level Warning + Write-Message "Creating empty matrix file..." + + # Create empty matrix + $emptyMatrix = @{ include = @() } + $outputFile = Join-Path $OutputDirectory "split-tests-matrix.json" + + # Ensure output directory exists + if (-not (Test-Path $OutputDirectory)) { + New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null + } + + $emptyMatrix | ConvertTo-Json -Depth 10 -Compress | Set-Content -Path $outputFile -Encoding UTF8 + Write-Message "Created empty matrix: $outputFile" -Level Success + exit 0 +} + +Write-Message "Found $($listFiles.Count) test list file(s)" -Level Success + +# Process each test list file +$allEntries = [System.Collections.ArrayList]::new() +$stats = @{} + +foreach ($listFile in $listFiles) { + # Extract project name (e.g., "Aspire.Templates.Tests.tests.list" → "Aspire.Templates.Tests") + $projectName = [System.IO.Path]::GetFileNameWithoutExtension($listFile.Name -replace '\.tests$', '') + + Write-Message "Processing $projectName..." + + # Read test class names + $classes = Get-Content $listFile.FullName | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } + + if ($classes.Count -eq 0) { + Write-Message " No test classes found, skipping" -Level Warning + continue + } + + # Read metadata + $metadataFile = $listFile.FullName -replace '\.tests\.list$', '.tests.metadata.json' + $metadata = Read-TestMetadata -MetadataFile $metadataFile -ProjectName $projectName + + # Generate matrix entry for each test class + $projectEntryCount = 0 + foreach ($class in $classes) { + $entry = New-MatrixEntry -FullClassName $class -ProjectName $projectName -Metadata $metadata + [void]$allEntries.Add($entry) + $projectEntryCount++ + } + + $stats[$projectName] = $projectEntryCount + Write-Message " Added $projectEntryCount test class(es)" -Level Success +} + +# Generate final matrix +$matrix = @{ + include = $allEntries.ToArray() +} + +# Write JSON file +$outputFile = Join-Path $OutputDirectory "split-tests-matrix.json" + +# Ensure output directory exists +if (-not (Test-Path $OutputDirectory)) { + New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null +} + +$jsonOutput = $matrix | ConvertTo-Json -Depth 10 -Compress +$jsonOutput | Set-Content -Path $outputFile -Encoding UTF8 -NoNewline + +Write-Message "" +Write-Message "Generated matrix with $($allEntries.Count) total test(s)" -Level Success +Write-Message "Output file: $outputFile" -Level Success +Write-Message "" +Write-Message "Matrix breakdown by project:" -Level Info + +foreach ($proj in $stats.Keys | Sort-Object) { + Write-Message " $proj`: $($stats[$proj]) class(es)" -Level Info +} + +Write-Message "" +Write-Message "Matrix generation complete! ✨" -Level Success + +#endregion +``` + +## Script Features + +### Cross-Platform Compatibility + +- ✅ Uses `System.IO.Path` for path operations +- ✅ No OS-specific cmdlets +- ✅ Tested on Windows, Linux, macOS +- ✅ UTF-8 encoding for JSON output + +### Error Handling + +- Validates input directory exists +- Handles missing metadata gracefully (uses defaults) +- Creates empty matrix if no tests found (CI won't fail) +- Detailed error messages + +### Logging + +- Color-coded output (Info, Success, Warning, Error) +- Shows progress per project +- Summary statistics at end +- Helpful for debugging CI issues + +## Testing the Script + +### Test 1: Empty Directory + +```powershell +# Should create empty matrix without errors +mkdir test-empty +pwsh eng/scripts/generate-test-matrix.ps1 ` + -TestListsDirectory ./test-empty ` + -OutputDirectory ./test-output ` + -BuildOs linux +``` + +**Expected**: Creates `split-tests-matrix.json` with `{"include":[]}` + +### Test 2: With Test Lists + +```powershell +# First, build a split test project to generate .tests.list files +dotnet build tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj ` + /t:Build;ExtractTestClassNames ` + -p:PrepareForHelix=true ` + -p:SplitTestsForCI=true ` + -p:TestClassNamesPrefix=Aspire.Templates.Tests + +# Then run the script +pwsh eng/scripts/generate-test-matrix.ps1 ` + -TestListsDirectory ./artifacts/helix ` + -OutputDirectory ./artifacts/test-matrices ` + -BuildOs linux +``` + +**Expected**: +- Creates matrix with ~10-15 entries +- Each entry has all required fields +- Valid JSON + +### Test 3: Verify JSON Structure + +```powershell +# Load and inspect the generated matrix +$matrix = Get-Content ./artifacts/test-matrices/split-tests-matrix.json | ConvertFrom-Json + +# Check structure +$matrix.include.Count # Should be > 0 +$matrix.include[0].PSObject.Properties.Name # Should show all fields + +# Verify required fields +$matrix.include | ForEach-Object { + if (-not $_.shortname) { Write-Error "Missing shortname" } + if (-not $_.fullClassName) { Write-Error "Missing fullClassName" } + if (-not $_.projectName) { Write-Error "Missing projectName" } +} +``` + +## Common Issues + +### Issue 1: "Cannot find path" + +**Cause**: TestListsDirectory doesn't exist +**Fix**: Ensure the directory is created before running script + +### Issue 2: Invalid JSON + +**Cause**: Special characters in class names +**Fix**: PowerShell's `ConvertTo-Json` handles this automatically + +### Issue 3: Empty matrix but tests exist + +**Cause**: `.tests.list` files not in expected location +**Fix**: Check `artifacts/helix/` directory structure + +## Next Steps + +Proceed to [Step 3: GitHub Actions Integration](./STEP_03_GITHUB_ACTIONS.md) \ No newline at end of file diff --git a/docs/test-splitting/STEP_02_POWERSHELL_SCRIPT_V2.md b/docs/test-splitting/STEP_02_POWERSHELL_SCRIPT_V2.md new file mode 100644 index 00000000000..5d92131f6fb --- /dev/null +++ b/docs/test-splitting/STEP_02_POWERSHELL_SCRIPT_V2.md @@ -0,0 +1,708 @@ +# Step 2: PowerShell Script Implementation (v2 - Collection Support) + +## Overview + +Enhanced PowerShell script that reads collection-based test lists and generates a matrix with: +- One entry per collection +- One entry for all uncollected tests + +## File: `eng/scripts/generate-test-matrix.ps1` + +### Complete Implementation + +```powershell +<# +.SYNOPSIS + Generates CI test matrices from collection-based test enumeration files. + +.DESCRIPTION + This script reads .tests.list and .tests.metadata.json files produced by the + ExtractTestClassNames MSBuild target and generates a JSON matrix file for + consumption by GitHub Actions or Azure DevOps. + + Supports both xUnit collections (grouped tests) and uncollected tests (catch-all). + + The script is cross-platform and runs on Windows, Linux, and macOS. + +.PARAMETER TestListsDirectory + Directory containing .tests.list and .tests.metadata.json files. + Typically: artifacts/helix/ + +.PARAMETER OutputDirectory + Directory where the JSON matrix file will be written. + Typically: artifacts/test-matrices/ + +.PARAMETER BuildOs + Current operating system being built for (windows, linux, darwin). + Used for logging and debugging. + +.EXAMPLE + pwsh generate-test-matrix.ps1 -TestListsDirectory ./artifacts/helix -OutputDirectory ./artifacts/matrices -BuildOs linux + +.NOTES + Author: Aspire Team + Date: 2025-10-16 + Version: 2.0 (Collection-based splitting) + Requires: PowerShell 7.0+ (cross-platform) +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory=$true, HelpMessage="Directory containing test list files")] + [ValidateScript({Test-Path $_ -PathType Container})] + [string]$TestListsDirectory, + + [Parameter(Mandatory=$true, HelpMessage="Output directory for matrix JSON")] + [string]$OutputDirectory, + + [Parameter(Mandatory=$false, HelpMessage="Current OS: windows, linux, or darwin")] + [ValidateSet('windows', 'linux', 'darwin', '')] + [string]$BuildOs = '' +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +#region Helper Functions + +function Write-Message { + <# + .SYNOPSIS + Writes a formatted message to the console. + #> + param( + [Parameter(Mandatory=$true)] + [AllowEmptyString()] + [string]$Message, + + [Parameter(Mandatory=$false)] + [ValidateSet('Info', 'Success', 'Warning', 'Error', 'Debug')] + [string]$Level = 'Info' + ) + + $prefix = switch ($Level) { + 'Success' { '✅' } + 'Warning' { '⚠️' } + 'Error' { '❌' } + 'Debug' { '🔍' } + default { 'ℹ️' } + } + + $color = switch ($Level) { + 'Success' { 'Green' } + 'Warning' { 'Yellow' } + 'Error' { 'Red' } + 'Debug' { 'Gray' } + default { 'Cyan' } + } + + Write-Host "$prefix $Message" -ForegroundColor $color +} + +function Get-TestListFiles { + <# + .SYNOPSIS + Finds all .tests.list files in the specified directory. + #> + param([string]$Directory) + + Get-ChildItem -Path $Directory -Filter "*.tests.list" -Recurse -ErrorAction SilentlyContinue +} + +function Read-TestMetadata { + <# + .SYNOPSIS + Reads and parses test metadata JSON file. + #> + param( + [string]$MetadataFile, + [string]$ProjectName + ) + + # Default metadata values + $defaults = @{ + testClassNamesPrefix = $ProjectName + testProjectPath = "tests/$ProjectName/$ProjectName.csproj" + collections = '' + requiresNugets = 'false' + requiresTestSdk = 'false' + testSessionTimeout = '20m' + testHangTimeout = '10m' + uncollectedTestsSessionTimeout = '15m' + uncollectedTestsHangTimeout = '8m' + enablePlaywrightInstall = 'false' + } + + if (-not (Test-Path $MetadataFile)) { + Write-Message "No metadata file found for $ProjectName, using defaults" -Level Warning + return $defaults + } + + try { + $content = Get-Content $MetadataFile -Raw | ConvertFrom-Json + + # Merge with defaults (content overrides defaults) + foreach ($key in $content.PSObject.Properties.Name) { + $defaults[$key] = $content.$key + } + + return $defaults + } + catch { + Write-Message "Failed to parse metadata for ${ProjectName}: $_" -Level Warning + return $defaults + } +} + +function Get-CollectionFilterArg { + <# + .SYNOPSIS + Generates xUnit filter argument for a specific collection. + #> + param([string]$CollectionName) + + return "--filter-collection `"$CollectionName`"" +} + +function Get-UncollectedFilterArg { + <# + .SYNOPSIS + Generates xUnit filter argument to exclude all collections. + #> + param([string[]]$Collections) + + if ($Collections.Count -eq 0) { + # No collections to exclude - run all tests + return "" + } + + # Build filter to exclude all collections + $filters = $Collections | ForEach-Object { + "--filter-not-collection `"$_`"" + } + + return $filters -join ' ' +} + +function New-CollectionMatrixEntry { + <# + .SYNOPSIS + Creates a matrix entry for a collection. + #> + param( + [string]$CollectionName, + [string]$ProjectName, + [hashtable]$Metadata + ) + + $filterArg = Get-CollectionFilterArg -CollectionName $CollectionName + + # Check for per-collection timeout overrides + $collectionTimeoutKey = "TestCollection_${CollectionName}_SessionTimeout" + $collectionHangTimeoutKey = "TestCollection_${CollectionName}_HangTimeout" + + $sessionTimeout = $Metadata.testSessionTimeout + $hangTimeout = $Metadata.testHangTimeout + + # Per-collection timeouts would come from metadata if specified + # For now, use project defaults + + [ordered]@{ + type = "collection" + name = $CollectionName + shortname = "Collection_$CollectionName" + projectName = $ProjectName + testProjectPath = $Metadata.testProjectPath + filterArg = $filterArg + requiresNugets = ($Metadata.requiresNugets -eq 'true') + requiresTestSdk = ($Metadata.requiresTestSdk -eq 'true') + testSessionTimeout = $sessionTimeout + testHangTimeout = $hangTimeout + enablePlaywrightInstall = ($Metadata.enablePlaywrightInstall -eq 'true') + } +} + +function New-UncollectedMatrixEntry { + <# + .SYNOPSIS + Creates a matrix entry for uncollected tests. + #> + param( + [string[]]$Collections, + [string]$ProjectName, + [hashtable]$Metadata + ) + + $filterArg = Get-UncollectedFilterArg -Collections $Collections + + # Use specific timeouts for uncollected tests (usually faster) + $sessionTimeout = if ($Metadata.uncollectedTestsSessionTimeout) { + $Metadata.uncollectedTestsSessionTimeout + } else { + $Metadata.testSessionTimeout + } + + $hangTimeout = if ($Metadata.uncollectedTestsHangTimeout) { + $Metadata.uncollectedTestsHangTimeout + } else { + $Metadata.testHangTimeout + } + + [ordered]@{ + type = "uncollected" + name = "UncollectedTests" + shortname = "Uncollected" + projectName = $ProjectName + testProjectPath = $Metadata.testProjectPath + filterArg = $filterArg + requiresNugets = ($Metadata.requiresNugets -eq 'true') + requiresTestSdk = ($Metadata.requiresTestSdk -eq 'true') + testSessionTimeout = $sessionTimeout + testHangTimeout = $hangTimeout + enablePlaywrightInstall = ($Metadata.enablePlaywrightInstall -eq 'true') + } +} + +function Parse-TestListFile { + <# + .SYNOPSIS + Parses a .tests.list file and returns collections and flags. + #> + param([string]$FilePath) + + $lines = Get-Content $FilePath | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } + + $result = @{ + Collections = [System.Collections.ArrayList]::new() + HasUncollected = $false + } + + foreach ($line in $lines) { + if ($line -match '^collection:(.+)$') { + [void]$result.Collections.Add($Matches[1].Trim()) + } + elseif ($line -match '^uncollected:') { + $result.HasUncollected = $true + } + } + + return $result +} + +#endregion + +#region Main Script + +Write-Message "Starting collection-based matrix generation for BuildOs=$BuildOs" +Write-Message "Test lists directory: $TestListsDirectory" +Write-Message "Output directory: $OutputDirectory" + +# Find all test list files +$listFiles = Get-TestListFiles -Directory $TestListsDirectory + +if ($listFiles.Count -eq 0) { + Write-Message "No test list files found in $TestListsDirectory" -Level Warning + Write-Message "Creating empty matrix file..." + + # Create empty matrix + $emptyMatrix = @{ include = @() } + $outputFile = Join-Path $OutputDirectory "split-tests-matrix.json" + + # Ensure output directory exists + if (-not (Test-Path $OutputDirectory)) { + New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null + } + + $emptyMatrix | ConvertTo-Json -Depth 10 -Compress | Set-Content -Path $outputFile -Encoding UTF8 + Write-Message "Created empty matrix: $outputFile" -Level Success + exit 0 +} + +Write-Message "Found $($listFiles.Count) test list file(s)" -Level Success + +# Process each test list file +$allEntries = [System.Collections.ArrayList]::new() +$stats = @{} + +foreach ($listFile in $listFiles) { + # Extract project name + $projectName = [System.IO.Path]::GetFileNameWithoutExtension($listFile.Name -replace '\.tests$', '') + + Write-Message "" + Write-Message "Processing $projectName..." -Level Info + + # Parse test list file + $parsed = Parse-TestListFile -FilePath $listFile.FullName + + if ($parsed.Collections.Count -eq 0 -and -not $parsed.HasUncollected) { + Write-Message " No collections or uncollected tests found, skipping" -Level Warning + continue + } + + # Read metadata + $metadataFile = $listFile.FullName -replace '\.tests\.list$', '.tests.metadata.json' + $metadata = Read-TestMetadata -MetadataFile $metadataFile -ProjectName $projectName + + $projectStats = @{ + Collections = 0 + Uncollected = 0 + } + + # Generate matrix entries for each collection + foreach ($collectionName in $parsed.Collections) { + Write-Message " Found collection: $collectionName" -Level Debug + + $entry = New-CollectionMatrixEntry ` + -CollectionName $collectionName ` + -ProjectName $projectName ` + -Metadata $metadata + + [void]$allEntries.Add($entry) + $projectStats.Collections++ + } + + # Generate matrix entry for uncollected tests + if ($parsed.HasUncollected) { + Write-Message " Adding uncollected tests job" -Level Debug + + $entry = New-UncollectedMatrixEntry ` + -Collections $parsed.Collections.ToArray() ` + -ProjectName $projectName ` + -Metadata $metadata + + [void]$allEntries.Add($entry) + $projectStats.Uncollected = 1 + } + + $stats[$projectName] = $projectStats + + $totalJobs = $projectStats.Collections + $projectStats.Uncollected + Write-Message " ✅ Generated $totalJobs job(s): $($projectStats.Collections) collection(s) + $($projectStats.Uncollected) uncollected" -Level Success +} + +# Generate final matrix +$matrix = @{ + include = $allEntries.ToArray() +} + +# Write JSON file +$outputFile = Join-Path $OutputDirectory "split-tests-matrix.json" + +# Ensure output directory exists +if (-not (Test-Path $OutputDirectory)) { + New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null +} + +$jsonOutput = $matrix | ConvertTo-Json -Depth 10 -Compress +$jsonOutput | Set-Content -Path $outputFile -Encoding UTF8 -NoNewline + +# Summary +Write-Message "" +Write-Message ("=" * 60) -Level Info +Write-Message "Matrix Generation Summary" -Level Success +Write-Message ("=" * 60) -Level Info +Write-Message "" +Write-Message "Total Jobs: $($allEntries.Count)" -Level Success +Write-Message "Output File: $outputFile" -Level Success +Write-Message "" +Write-Message "Breakdown by Project:" -Level Info + +foreach ($proj in $stats.Keys | Sort-Object) { + $s = $stats[$proj] + $collText = if ($s.Collections -eq 1) { "collection" } else { "collections" } + $uncText = if ($s.Uncollected -eq 1) { "uncollected job" } else { "uncollected jobs" } + + Write-Message " $proj`: $($s.Collections) $collText + $($s.Uncollected) $uncText" -Level Info +} + +Write-Message "" +Write-Message "Matrix generation complete! ✨" -Level Success + +#endregion +``` + +## Key Features of v2 Script + +### 1. Collection Parsing + +```powershell +function Parse-TestListFile { + # Parses format: + # collection:DatabaseTests + # collection:ContainerTests + # uncollected:* + + foreach ($line in $lines) { + if ($line -match '^collection:(.+)$') { + # Extract collection name + } + elseif ($line -match '^uncollected:') { + # Flag that uncollected tests exist + } + } +} +``` + +### 2. Filter Generation + +```powershell +# For a collection +"--filter-collection `"DatabaseTests`"" + +# For uncollected (exclude all collections) +"--filter-not-collection `"DatabaseTests`" --filter-not-collection `"ContainerTests`"" +``` + +### 3. Smart Timeouts + +```powershell +# Collections use project-level timeouts (usually longer) +$sessionTimeout = $Metadata.testSessionTimeout # e.g., 25m + +# Uncollected uses shorter timeouts (fast tests) +$sessionTimeout = $Metadata.uncollectedTestsSessionTimeout # e.g., 15m +``` + +### 4. Matrix Entry Types + +```powershell +# Collection entry +@{ + type = "collection" + name = "DatabaseTests" + filterArg = "--filter-collection `"DatabaseTests`"" + # ... +} + +# Uncollected entry +@{ + type = "uncollected" + name = "UncollectedTests" + filterArg = "--filter-not-collection `"DatabaseTests`" ..." + # ... +} +``` + +## Testing the Script + +### Test 1: Project with No Collections + +Create a test list file: + +```bash +# artifacts/helix/SomeProject.Tests.tests.list +uncollected:* +``` + +Create metadata: + +```json +{ + "projectName": "SomeProject.Tests", + "testProjectPath": "tests/SomeProject.Tests/SomeProject.Tests.csproj", + "collections": "", + "testSessionTimeout": "20m", + "testHangTimeout": "10m", + "uncollectedTestsSessionTimeout": "15m", + "uncollectedTestsHangTimeout": "8m" +} +``` + +Run script: + +```powershell +pwsh eng/scripts/generate-test-matrix.ps1 ` + -TestListsDirectory ./artifacts/helix ` + -OutputDirectory ./artifacts/test-matrices ` + -BuildOs linux +``` + +**Expected Output**: +```json +{ + "include": [ + { + "type": "uncollected", + "name": "UncollectedTests", + "shortname": "Uncollected", + "projectName": "SomeProject.Tests", + "testProjectPath": "tests/SomeProject.Tests/SomeProject.Tests.csproj", + "filterArg": "", + "requiresNugets": false, + "requiresTestSdk": false, + "testSessionTimeout": "15m", + "testHangTimeout": "8m", + "enablePlaywrightInstall": false + } + ] +} +``` + +**Result**: 1 job + +### Test 2: Project with Collections + +Create test list: + +```bash +# artifacts/helix/Aspire.Hosting.Tests.tests.list +collection:DatabaseTests +collection:ContainerTests +uncollected:* +``` + +Create metadata: + +```json +{ + "projectName": "Aspire.Hosting.Tests", + "testProjectPath": "tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj", + "collections": "DatabaseTests;ContainerTests", + "testSessionTimeout": "25m", + "testHangTimeout": "12m", + "uncollectedTestsSessionTimeout": "15m", + "uncollectedTestsHangTimeout": "8m", + "requiresNugets": "false", + "requiresTestSdk": "false", + "enablePlaywrightInstall": "false" +} +``` + +Run script: + +```powershell +pwsh eng/scripts/generate-test-matrix.ps1 ` + -TestListsDirectory ./artifacts/helix ` + -OutputDirectory ./artifacts/test-matrices ` + -BuildOs linux +``` + +**Expected Output**: + +```json +{ + "include": [ + { + "type": "collection", + "name": "DatabaseTests", + "shortname": "Collection_DatabaseTests", + "projectName": "Aspire.Hosting.Tests", + "filterArg": "--filter-collection \"DatabaseTests\"", + "testSessionTimeout": "25m", + "testHangTimeout": "12m", + ... + }, + { + "type": "collection", + "name": "ContainerTests", + "shortname": "Collection_ContainerTests", + "projectName": "Aspire.Hosting.Tests", + "filterArg": "--filter-collection \"ContainerTests\"", + "testSessionTimeout": "25m", + "testHangTimeout": "12m", + ... + }, + { + "type": "uncollected", + "name": "UncollectedTests", + "shortname": "Uncollected", + "projectName": "Aspire.Hosting.Tests", + "filterArg": "--filter-not-collection \"DatabaseTests\" --filter-not-collection \"ContainerTests\"", + "testSessionTimeout": "15m", + "testHangTimeout": "8m", + ... + } + ] +} +``` + +**Result**: 3 jobs + +### Test 3: Verify Filter Arguments + +Load and inspect the matrix: + +```powershell +$matrix = Get-Content ./artifacts/test-matrices/split-tests-matrix.json | ConvertFrom-Json + +# Check collection filters +$matrix.include | Where-Object { $_.type -eq 'collection' } | ForEach-Object { + Write-Host "$($_.name): $($_.filterArg)" +} + +# Check uncollected filter +$uncollected = $matrix.include | Where-Object { $_.type -eq 'uncollected' } +Write-Host "Uncollected: $($uncollected.filterArg)" +``` + +**Expected Console Output**: +``` +DatabaseTests: --filter-collection "DatabaseTests" +ContainerTests: --filter-collection "ContainerTests" +Uncollected: --filter-not-collection "DatabaseTests" --filter-not-collection "ContainerTests" +``` + +### Test 4: Multiple Projects + +Create test lists for multiple projects: + +```bash +# artifacts/helix/Aspire.Hosting.Tests.tests.list +collection:DatabaseTests +uncollected:* + +# artifacts/helix/Aspire.Templates.Tests.tests.list +collection:StarterTemplate +collection:BasicTemplate +uncollected:* +``` + +Run script: + +```powershell +pwsh eng/scripts/generate-test-matrix.ps1 ` + -TestListsDirectory ./artifacts/helix ` + -OutputDirectory ./artifacts/test-matrices ` + -BuildOs linux +``` + +**Expected Result**: 6 jobs total +- 2 from Aspire.Hosting.Tests (1 collection + 1 uncollected) +- 4 from Aspire.Templates.Tests (2 collections + 1 uncollected) + +## Validation Checklist + +- [ ] Script runs without errors on all 3 OSes +- [ ] Empty directory creates empty matrix +- [ ] Single uncollected entry creates 1 job +- [ ] Collections create separate jobs +- [ ] Uncollected filter excludes all collections +- [ ] Metadata defaults work when file missing +- [ ] JSON output is valid and parseable +- [ ] Filter arguments have correct syntax +- [ ] Timeouts are applied correctly +- [ ] Summary statistics are accurate + +## Common Issues & Solutions + +### Issue 1: "Collection not found" in test output + +**Symptom**: xunit can't find collection name +**Cause**: Collection name has special characters or spaces +**Fix**: Escape collection names in filter arguments (already handled with quotes) + +### Issue 2: Uncollected filter too long + +**Symptom**: Command line too long with many collections +**Cause**: Too many `--filter-not-collection` arguments +**Fix**: Consider regrouping collections or using different approach + +### Issue 3: Empty uncollected job + +**Symptom**: Uncollected job runs but no tests execute +**Cause**: All tests are in collections +**Fix**: This is OK - job will exit with code 8 (zero tests), which we ignore + +## Next Steps + +Proceed to [Step 4: Project Configuration (v2)](./STEP_04_PROJECT_CONFIG_V2.md) - GitHub Actions doesn't need changes since it just consumes the matrix! \ No newline at end of file diff --git a/docs/test-splitting/STEP_03_GITHUB_ACTIONS.md b/docs/test-splitting/STEP_03_GITHUB_ACTIONS.md new file mode 100644 index 00000000000..78d70269c81 --- /dev/null +++ b/docs/test-splitting/STEP_03_GITHUB_ACTIONS.md @@ -0,0 +1,414 @@ +# Step 3: GitHub Actions Integration + +## Overview + +Update GitHub Actions workflows to use the new MSBuild-based matrix generation while maintaining full support for all 3 OSes. + +## Critical Requirement: Per-OS Matrix Generation + +**Each OS MUST generate its own matrix** because: +1. Projects can opt-in/out per OS (`RunOnGithubActionsWindows`, etc.) +2. Some tests only run on specific OSes (e.g., Docker tests on Linux) +3. File path differences between OSes +4. Test discovery may differ per platform + +## File: `.github/actions/enumerate-tests/action.yml` + +### Complete Replacement + +```yaml +name: 'Enumerate test projects' +description: 'Enumerate test projects and generate test matrices for the current OS' +inputs: + includeIntegrations: + description: 'Include integration tests in enumeration' + required: false + type: boolean + default: false + includeSplitTests: + description: 'Include and generate split test matrices' + required: false + type: boolean + default: false + +outputs: + integrations_tests_matrix: + description: 'JSON matrix of integration test projects' + value: ${{ steps.load_integrations_matrix.outputs.matrix }} + split_tests_matrix: + description: 'JSON matrix of split test classes' + value: ${{ steps.load_split_matrix.outputs.matrix }} + +runs: + using: "composite" + steps: + - name: Checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Set up .NET Core + uses: actions/setup-dotnet@3951f0dfe7a07e2313ec93c75700083e2005cbab # v4.3.0 + with: + global-json-file: ${{ github.workspace }}/global.json + + - name: Generate test project lists + if: ${{ inputs.includeIntegrations }} + shell: pwsh + run: > + dotnet build ${{ github.workspace }}/tests/Shared/GetTestProjects.proj + /bl:${{ github.workspace }}/artifacts/log/Debug/GetTestProjects.binlog + /p:TestsListOutputPath=${{ github.workspace }}/artifacts/TestsForGithubActions.list + /p:TestMatrixOutputPath=${{ github.workspace }}/artifacts/test-matrices/ + /p:ContinuousIntegrationBuild=true + + - name: Build split test projects + if: ${{ inputs.includeSplitTests }} + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + + $splitProjectsFile = "${{ github.workspace }}/artifacts/TestsForGithubActions.list.split-projects" + + if (-not (Test-Path $splitProjectsFile)) { + Write-Host "::notice::No split test projects found for ${{ runner.os }}" + exit 0 + } + + $splitProjects = Get-Content $splitProjectsFile | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } + + if ($splitProjects.Count -eq 0) { + Write-Host "::notice::No split test projects to build for ${{ runner.os }}" + exit 0 + } + + Write-Host "::group::Building $($splitProjects.Count) split test project(s) for ${{ runner.os }}" + + foreach ($shortname in $splitProjects) { + Write-Host "Processing $shortname..." + + # Find the project file (try both naming patterns) + $projectPath1 = "${{ github.workspace }}/tests/$shortname.Tests/$shortname.Tests.csproj" + $projectPath2 = "${{ github.workspace }}/tests/Aspire.$shortname.Tests/Aspire.$shortname.Tests.csproj" + + if (Test-Path $projectPath1) { + $projectPath = $projectPath1 + } elseif (Test-Path $projectPath2) { + $projectPath = $projectPath2 + } else { + Write-Error "::error::Could not find project for $shortname" + exit 1 + } + + Write-Host " Building: $projectPath" + + # Build with ExtractTestClassNames target + dotnet build $projectPath ` + /t:Build`;ExtractTestClassNames ` + /bl:${{ github.workspace }}/artifacts/log/Debug/Build_$shortname.binlog ` + -p:PrepareForHelix=true ` + -p:SplitTestsForCI=true ` + -p:InstallBrowsersForPlaywright=false + + if ($LASTEXITCODE -ne 0) { + Write-Error "::error::Build failed for $shortname with exit code $LASTEXITCODE" + exit $LASTEXITCODE + } + + Write-Host " ✅ Successfully built $shortname" + } + + Write-Host "::endgroup::" + Write-Host "::notice::Successfully built all $($splitProjects.Count) split test projects for ${{ runner.os }}" + + - name: Load integrations matrix + id: load_integrations_matrix + if: ${{ inputs.includeIntegrations }} + shell: pwsh + run: | + $filePath = "${{ github.workspace }}/artifacts/TestsForGithubActions.list" + + if (-not (Test-Path $filePath)) { + Write-Error "::error::Test list file not found: $filePath" + exit 1 + } + + $lines = Get-Content $filePath | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } + + $matrix = @{ + shortname = $lines | Sort-Object + } + + $json = $matrix | ConvertTo-Json -Compress + + Write-Host "::notice::Generated integrations matrix for ${{ runner.os }} with $($lines.Count) project(s)" + "matrix=$json" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + + - name: Load split tests matrix + id: load_split_matrix + if: ${{ inputs.includeSplitTests }} + shell: pwsh + run: | + $matrixFile = "${{ github.workspace }}/artifacts/test-matrices/split-tests-matrix.json" + + if (Test-Path $matrixFile) { + $json = Get-Content $matrixFile -Raw + $matrix = $json | ConvertFrom-Json + + $testCount = if ($matrix.include) { $matrix.include.Count } else { 0 } + + Write-Host "::notice::Generated split tests matrix for ${{ runner.os }} with $testCount test(s)" + "matrix=$json" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + } else { + Write-Host "::notice::No split tests matrix found for ${{ runner.os }}, using empty matrix" + $emptyMatrix = @{ include = @() } | ConvertTo-Json -Compress + "matrix=$emptyMatrix" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + } + + - name: Upload artifacts + if: always() + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: logs-enumerate-tests-${{ runner.os }} + path: | + artifacts/log/**/*.binlog + artifacts/**/*.list + artifacts/**/*.metadata.json + artifacts/test-matrices/**/*.json + if-no-files-found: warn +``` + +## File: `.github/workflows/tests.yml` + +### Modified Sections + +#### 1. Update setup jobs (KEEP SEPARATE PER OS) + +```yaml +jobs: + # IMPORTANT: Keep separate setup jobs for each OS + # Each OS generates its own matrix because projects can opt-in/out per OS + + setup_for_tests_lin: + name: Setup for tests (Linux) + runs-on: ubuntu-latest + outputs: + integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }} + split_tests_matrix: ${{ steps.generate_tests_matrix.outputs.split_tests_matrix }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - uses: ./.github/actions/enumerate-tests + id: generate_tests_matrix + with: + includeIntegrations: true + includeSplitTests: true # NEW: Enable split tests + + setup_for_tests_macos: + name: Setup for tests (macOS) + runs-on: macos-latest + outputs: + integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }} + split_tests_matrix: ${{ steps.generate_tests_matrix.outputs.split_tests_matrix }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - uses: ./.github/actions/enumerate-tests + id: generate_tests_matrix + with: + includeIntegrations: true + includeSplitTests: true # NEW: Enable split tests + + setup_for_tests_win: + name: Setup for tests (Windows) + runs-on: windows-latest + outputs: + integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }} + split_tests_matrix: ${{ steps.generate_tests_matrix.outputs.split_tests_matrix }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - uses: ./.github/actions/enumerate-tests + id: generate_tests_matrix + with: + includeIntegrations: true + includeSplitTests: true # NEW: Enable split tests +``` + +#### 2. Add split test jobs (NEW) + +```yaml + # NEW: Split tests for Linux + split_tests_lin: + uses: ./.github/workflows/run-tests.yml + name: Split Tests Linux + needs: [setup_for_tests_lin, build_packages] + if: ${{ fromJson(needs.setup_for_tests_lin.outputs.split_tests_matrix).include[0] != null }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup_for_tests_lin.outputs.split_tests_matrix) }} + with: + testShortName: "${{ matrix.projectName }}_${{ matrix.shortname }}" + testProjectPath: "${{ matrix.testProjectPath }}" + os: "ubuntu-latest" + testSessionTimeout: "${{ matrix.testSessionTimeout }}" + testHangTimeout: "${{ matrix.testHangTimeout }}" + extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true --filter-class ${{ matrix.fullClassName }}" + requiresNugets: ${{ matrix.requiresNugets }} + requiresTestSdk: ${{ matrix.requiresTestSdk }} + enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} + versionOverrideArg: ${{ inputs.versionOverrideArg }} + + # NEW: Split tests for macOS + split_tests_macos: + uses: ./.github/workflows/run-tests.yml + name: Split Tests macOS + needs: [setup_for_tests_macos, build_packages] + if: ${{ fromJson(needs.setup_for_tests_macos.outputs.split_tests_matrix).include[0] != null }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup_for_tests_macos.outputs.split_tests_matrix) }} + with: + testShortName: "${{ matrix.projectName }}_${{ matrix.shortname }}" + testProjectPath: "${{ matrix.testProjectPath }}" + os: "macos-latest" + testSessionTimeout: "${{ matrix.testSessionTimeout }}" + testHangTimeout: "${{ matrix.testHangTimeout }}" + extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true --filter-class ${{ matrix.fullClassName }}" + requiresNugets: ${{ matrix.requiresNugets }} + requiresTestSdk: ${{ matrix.requiresTestSdk }} + enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} + versionOverrideArg: ${{ inputs.versionOverrideArg }} + + # NEW: Split tests for Windows + split_tests_win: + uses: ./.github/workflows/run-tests.yml + name: Split Tests Windows + needs: [setup_for_tests_win, build_packages] + if: ${{ fromJson(needs.setup_for_tests_win.outputs.split_tests_matrix).include[0] != null }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup_for_tests_win.outputs.split_tests_matrix) }} + with: + testShortName: "${{ matrix.projectName }}_${{ matrix.shortname }}" + testProjectPath: "${{ matrix.testProjectPath }}" + os: "windows-latest" + testSessionTimeout: "${{ matrix.testSessionTimeout }}" + testHangTimeout: "${{ matrix.testHangTimeout }}" + extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true --filter-class ${{ matrix.fullClassName }}" + requiresNugets: ${{ matrix.requiresNugets }} + requiresTestSdk: ${{ matrix.requiresTestSdk }} + enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} + versionOverrideArg: ${{ inputs.versionOverrideArg }} +``` + +#### 3. REMOVE old templates_test_* jobs + +```yaml +# DELETE THESE (they'll use the new split_tests_* jobs instead): +# - templates_test_lin +# - templates_test_macos +# - templates_test_win +``` + +#### 4. Update results job dependencies + +```yaml + results: + if: ${{ always() && github.repository_owner == 'dotnet' }} + runs-on: ubuntu-latest + name: Final Test Results + needs: [ + endtoend_tests, + extension_tests_win, + integrations_test_lin, + integrations_test_macos, + integrations_test_win, + split_tests_lin, # NEW + split_tests_macos, # NEW + split_tests_win # NEW + ] + # ... rest of job unchanged ... +``` + +## Testing the Workflow Changes + +### Test 1: Dry Run with Empty Matrix + +Before enabling any split tests, verify the workflow handles empty matrices: + +1. Don't set `SplitTestsForCI=true` in any project +2. Push to a branch +3. Verify workflow runs successfully +4. Check that split_tests_* jobs are skipped (due to `if` condition) + +### Test 2: Enable for One Project + +1. Enable splitting for Aspire.Templates.Tests (already configured) +2. Push to a branch +3. Verify: + - 3 setup jobs run (one per OS) + - Each generates a matrix + - Split test jobs run in parallel + - Each test class runs separately + +### Test 3: Verify OS-Specific Matrices + +Check that each OS can have different matrices: + +1. Set a project to `RunOnGithubActionsLinux=true` but `RunOnGithubActionsWindows=false` +2. Verify Linux matrix includes it, Windows matrix doesn't +3. Verify Windows split_tests_win job is skipped or has fewer tests + +## Important Notes + +### Why Per-OS Setup Jobs? + +```yaml +# ❌ DON'T DO THIS - Single setup job +setup_for_tests: + runs-on: ubuntu-latest # Only Linux! + # This would only detect Linux tests + +# ✅ DO THIS - Per-OS setup jobs +setup_for_tests_lin: + runs-on: ubuntu-latest + +setup_for_tests_macos: + runs-on: macos-latest + +setup_for_tests_win: + runs-on: windows-latest +``` + +### Matrix Conditional + +The `if` condition prevents job failure when matrix is empty: + +```yaml +if: ${{ fromJson(needs.setup_for_tests_lin.outputs.split_tests_matrix).include[0] != null }} +``` + +This checks if the matrix has at least one entry. + +## Common Issues + +### Issue: "Invalid matrix" + +**Symptom**: Workflow fails with matrix parsing error +**Cause**: Malformed JSON from PowerShell script +**Fix**: Check `artifacts/test-matrices/split-tests-matrix.json` structure + +### Issue: Split tests not running + +**Symptom**: split_tests_* jobs are skipped +**Cause**: Empty matrix or missing `includeSplitTests: true` +**Fix**: Verify enumerate-tests action has correct inputs + +### Issue: Tests run on wrong OS + +**Symptom**: Linux tests running on Windows +**Cause**: Using wrong matrix output +**Fix**: Ensure each job uses the correct `needs.setup_for_tests_{os}.outputs` + +## Next Steps + +Proceed to [Step 4: Project Configuration](./STEP_04_PROJECT_CONFIG.md) \ No newline at end of file diff --git a/docs/test-splitting/STEP_03_MATRIX_GENERATOR_V3.md b/docs/test-splitting/STEP_03_MATRIX_GENERATOR_V3.md new file mode 100644 index 00000000000..1bc65f3c128 --- /dev/null +++ b/docs/test-splitting/STEP_03_MATRIX_GENERATOR_V3.md @@ -0,0 +1,865 @@ +# Step 3: Matrix Generator Implementation (v3 - Dual Mode Support) + +## Overview + +Enhanced PowerShell script that reads the auto-detected test lists and generates matrices for both collection-based and class-based splitting modes. + +## File: `eng/scripts/generate-test-matrix.ps1` + +### Complete Implementation + +```powershell +<# +.SYNOPSIS + Generates CI test matrices from auto-detected test enumeration files. + +.DESCRIPTION + This script reads .tests.list and .tests.metadata.json files and generates + a JSON matrix file for consumption by GitHub Actions or Azure DevOps. + + Automatically handles both modes: + - Collection-based: Entries like "collection:Name" and "uncollected:*" + - Class-based: Entries like "class:Full.Class.Name" + + The script is cross-platform and runs on Windows, Linux, and macOS. + +.PARAMETER TestListsDirectory + Directory containing .tests.list and .tests.metadata.json files. + Typically: artifacts/helix/ + +.PARAMETER OutputDirectory + Directory where the JSON matrix file will be written. + Typically: artifacts/test-matrices/ + +.PARAMETER BuildOs + Current operating system being built for (windows, linux, darwin). + Used for logging and debugging. + +.EXAMPLE + pwsh generate-test-matrix.ps1 -TestListsDirectory ./artifacts/helix -OutputDirectory ./artifacts/matrices -BuildOs linux + +.NOTES + Author: Aspire Team + Date: 2025-10-16 + Version: 3.0 (Auto-detection support) + Requires: PowerShell 7.0+ (cross-platform) +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory=$true, HelpMessage="Directory containing test list files")] + [ValidateScript({Test-Path $_ -PathType Container})] + [string]$TestListsDirectory, + + [Parameter(Mandatory=$true, HelpMessage="Output directory for matrix JSON")] + [string]$OutputDirectory, + + [Parameter(Mandatory=$false, HelpMessage="Current OS: windows, linux, or darwin")] + [ValidateSet('windows', 'linux', 'darwin', '')] + [string]$BuildOs = '' +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +#region Helper Functions + +function Write-Message { + <# + .SYNOPSIS + Writes a formatted message to the console. + #> + param( + [Parameter(Mandatory=$true)] + [AllowEmptyString()] + [string]$Message, + + [Parameter(Mandatory=$false)] + [ValidateSet('Info', 'Success', 'Warning', 'Error', 'Debug')] + [string]$Level = 'Info' + ) + + $prefix = switch ($Level) { + 'Success' { '✅' } + 'Warning' { '⚠️' } + 'Error' { '❌' } + 'Debug' { '🔍' } + default { 'ℹ️' } + } + + $color = switch ($Level) { + 'Success' { 'Green' } + 'Warning' { 'Yellow' } + 'Error' { 'Red' } + 'Debug' { 'Gray' } + default { 'Cyan' } + } + + Write-Host "$prefix $Message" -ForegroundColor $color +} + +function Get-TestListFiles { + <# + .SYNOPSIS + Finds all .tests.list files in the specified directory. + #> + param([string]$Directory) + + Get-ChildItem -Path $Directory -Filter "*.tests.list" -Recurse -ErrorAction SilentlyContinue +} + +function Read-TestMetadata { + <# + .SYNOPSIS + Reads and parses test metadata JSON file. + #> + param( + [string]$MetadataFile, + [string]$ProjectName + ) + + # Default metadata values + $defaults = @{ + projectName = $ProjectName + testClassNamesPrefix = $ProjectName + testProjectPath = "tests/$ProjectName/$ProjectName.csproj" + mode = 'class' + collections = '' + requiresNugets = 'false' + requiresTestSdk = 'false' + testSessionTimeout = '20m' + testHangTimeout = '10m' + uncollectedTestsSessionTimeout = '15m' + uncollectedTestsHangTimeout = '8m' + enablePlaywrightInstall = 'false' + } + + if (-not (Test-Path $MetadataFile)) { + Write-Message "No metadata file found for $ProjectName, using defaults" -Level Warning + return $defaults + } + + try { + $content = Get-Content $MetadataFile -Raw | ConvertFrom-Json + + # Merge with defaults (content overrides defaults) + foreach ($key in $content.PSObject.Properties.Name) { + $defaults[$key] = $content.$key + } + + return $defaults + } + catch { + Write-Message "Failed to parse metadata for ${ProjectName}: $_" -Level Warning + return $defaults + } +} + +function Get-CollectionFilterArg { + <# + .SYNOPSIS + Generates xUnit filter argument for a specific collection. + #> + param([string]$CollectionName) + + return "--filter-collection `"$CollectionName`"" +} + +function Get-UncollectedFilterArg { + <# + .SYNOPSIS + Generates xUnit filter argument to exclude all collections. + #> + param([string[]]$Collections) + + if ($Collections.Count -eq 0) { + # No collections to exclude - run all tests + return "" + } + + # Build filter to exclude all collections + $filters = $Collections | ForEach-Object { + "--filter-not-collection `"$_`"" + } + + return $filters -join ' ' +} + +function Get-ClassFilterArg { + <# + .SYNOPSIS + Generates xUnit filter argument for a specific test class. + #> + param([string]$ClassName) + + return "--filter-class `"$ClassName`"" +} + +function New-CollectionMatrixEntry { + <# + .SYNOPSIS + Creates a matrix entry for a collection. + #> + param( + [string]$CollectionName, + [string]$ProjectName, + [hashtable]$Metadata + ) + + $filterArg = Get-CollectionFilterArg -CollectionName $CollectionName + + [ordered]@{ + type = "collection" + name = $CollectionName + shortname = "Collection_$CollectionName" + projectName = $ProjectName + testProjectPath = $Metadata.testProjectPath + filterArg = $filterArg + requiresNugets = ($Metadata.requiresNugets -eq 'true') + requiresTestSdk = ($Metadata.requiresTestSdk -eq 'true') + testSessionTimeout = $Metadata.testSessionTimeout + testHangTimeout = $Metadata.testHangTimeout + enablePlaywrightInstall = ($Metadata.enablePlaywrightInstall -eq 'true') + } +} + +function New-UncollectedMatrixEntry { + <# + .SYNOPSIS + Creates a matrix entry for uncollected tests. + #> + param( + [string[]]$Collections, + [string]$ProjectName, + [hashtable]$Metadata + ) + + $filterArg = Get-UncollectedFilterArg -Collections $Collections + + # Use specific timeouts for uncollected tests (usually faster) + $sessionTimeout = if ($Metadata.uncollectedTestsSessionTimeout) { + $Metadata.uncollectedTestsSessionTimeout + } else { + $Metadata.testSessionTimeout + } + + $hangTimeout = if ($Metadata.uncollectedTestsHangTimeout) { + $Metadata.uncollectedTestsHangTimeout + } else { + $Metadata.testHangTimeout + } + + [ordered]@{ + type = "uncollected" + name = "UncollectedTests" + shortname = "Uncollected" + projectName = $ProjectName + testProjectPath = $Metadata.testProjectPath + filterArg = $filterArg + requiresNugets = ($Metadata.requiresNugets -eq 'true') + requiresTestSdk = ($Metadata.requiresTestSdk -eq 'true') + testSessionTimeout = $sessionTimeout + testHangTimeout = $hangTimeout + enablePlaywrightInstall = ($Metadata.enablePlaywrightInstall -eq 'true') + } +} + +function New-ClassMatrixEntry { + <# + .SYNOPSIS + Creates a matrix entry for a test class. + #> + param( + [string]$FullClassName, + [string]$ProjectName, + [hashtable]$Metadata + ) + + $prefix = $Metadata.testClassNamesPrefix + $shortname = $FullClassName + + # Strip prefix if present (e.g., "Aspire.Templates.Tests.MyClass" → "MyClass") + if ($prefix -and $FullClassName.StartsWith("$prefix.")) { + $shortname = $FullClassName.Substring($prefix.Length + 1) + } + + $filterArg = Get-ClassFilterArg -ClassName $FullClassName + + [ordered]@{ + type = "class" + fullClassName = $FullClassName + shortname = $shortname + projectName = $ProjectName + testProjectPath = $Metadata.testProjectPath + filterArg = $filterArg + requiresNugets = ($Metadata.requiresNugets -eq 'true') + requiresTestSdk = ($Metadata.requiresTestSdk -eq 'true') + testSessionTimeout = $Metadata.testSessionTimeout + testHangTimeout = $Metadata.testHangTimeout + enablePlaywrightInstall = ($Metadata.enablePlaywrightInstall -eq 'true') + } +} + +function Parse-TestListFile { + <# + .SYNOPSIS + Parses a .tests.list file and returns structured data. + #> + param([string]$FilePath) + + $lines = Get-Content $FilePath | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } + + $result = @{ + Mode = 'unknown' + Collections = [System.Collections.ArrayList]::new() + Classes = [System.Collections.ArrayList]::new() + HasUncollected = $false + } + + foreach ($line in $lines) { + if ($line -match '^collection:(.+)$') { + $result.Mode = 'collection' + [void]$result.Collections.Add($Matches[1].Trim()) + } + elseif ($line -match '^uncollected:') { + $result.HasUncollected = $true + } + elseif ($line -match '^class:(.+)$') { + $result.Mode = 'class' + [void]$result.Classes.Add($Matches[1].Trim()) + } + } + + return $result +} + +#endregion + +#region Main Script + +Write-Message "Starting matrix generation for BuildOs=$BuildOs" -Level Success +Write-Message "Test lists directory: $TestListsDirectory" +Write-Message "Output directory: $OutputDirectory" +Write-Message "" + +# Find all test list files +$listFiles = Get-TestListFiles -Directory $TestListsDirectory + +if ($listFiles.Count -eq 0) { + Write-Message "No test list files found in $TestListsDirectory" -Level Warning + Write-Message "Creating empty matrix file..." + + # Create empty matrix + $emptyMatrix = @{ include = @() } + $outputFile = Join-Path $OutputDirectory "split-tests-matrix.json" + + # Ensure output directory exists + if (-not (Test-Path $OutputDirectory)) { + New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null + } + + $emptyMatrix | ConvertTo-Json -Depth 10 -Compress | Set-Content -Path $outputFile -Encoding UTF8 + Write-Message "Created empty matrix: $outputFile" -Level Success + exit 0 +} + +Write-Message "Found $($listFiles.Count) test list file(s)" -Level Success +Write-Message "" + +# Process each test list file +$allEntries = [System.Collections.ArrayList]::new() +$stats = @{} + +foreach ($listFile in $listFiles) { + # Extract project name + $projectName = [System.IO.Path]::GetFileNameWithoutExtension($listFile.Name -replace '\.tests$', '') + + Write-Message "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -Level Info + Write-Message "Processing: $projectName" -Level Info + Write-Message "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -Level Info + + # Parse test list file + $parsed = Parse-TestListFile -FilePath $listFile.FullName + + if ($parsed.Mode -eq 'unknown') { + Write-Message " Unable to determine mode, skipping" -Level Warning + continue + } + + # Read metadata + $metadataFile = $listFile.FullName -replace '\.tests\.list$', '.tests.metadata.json' + $metadata = Read-TestMetadata -MetadataFile $metadataFile -ProjectName $projectName + + Write-Message " Mode: $($parsed.Mode)" -Level Info + + $projectStats = @{ + Mode = $parsed.Mode + Collections = 0 + Classes = 0 + Uncollected = 0 + } + + if ($parsed.Mode -eq 'collection') { + # Collection-based mode + Write-Message " Strategy: Collection-based splitting" -Level Success + Write-Message "" + + # Generate matrix entries for each collection + foreach ($collectionName in $parsed.Collections) { + Write-Message " ➕ Collection: $collectionName" -Level Debug + + $entry = New-CollectionMatrixEntry ` + -CollectionName $collectionName ` + -ProjectName $projectName ` + -Metadata $metadata + + [void]$allEntries.Add($entry) + $projectStats.Collections++ + } + + # Generate matrix entry for uncollected tests + if ($parsed.HasUncollected) { + Write-Message " ➕ Uncollected tests (all non-collection tests)" -Level Debug + + $entry = New-UncollectedMatrixEntry ` + -Collections $parsed.Collections.ToArray() ` + -ProjectName $projectName ` + -Metadata $metadata + + [void]$allEntries.Add($entry) + $projectStats.Uncollected = 1 + } + + $totalJobs = $projectStats.Collections + $projectStats.Uncollected + Write-Message "" + Write-Message " ✅ Generated $totalJobs job(s): $($projectStats.Collections) collection(s) + $($projectStats.Uncollected) uncollected" -Level Success + } + else { + # Class-based mode + Write-Message " Strategy: Class-based splitting" -Level Success + Write-Message "" + + # Generate matrix entries for each class + foreach ($className in $parsed.Classes) { + $shortName = $className -replace "^$($metadata.testClassNamesPrefix)\.", "" + Write-Message " ➕ Class: $shortName" -Level Debug + + $entry = New-ClassMatrixEntry ` + -FullClassName $className ` + -ProjectName $projectName ` + -Metadata $metadata + + [void]$allEntries.Add($entry) + $projectStats.Classes++ + } + + Write-Message "" + Write-Message " ✅ Generated $($projectStats.Classes) job(s): one per class" -Level Success + } + + $stats[$projectName] = $projectStats + Write-Message "" +} + +# Generate final matrix +$matrix = @{ + include = $allEntries.ToArray() +} + +# Write JSON file +$outputFile = Join-Path $OutputDirectory "split-tests-matrix.json" + +# Ensure output directory exists +if (-not (Test-Path $OutputDirectory)) { + New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null +} + +$jsonOutput = $matrix | ConvertTo-Json -Depth 10 -Compress +$jsonOutput | Set-Content -Path $outputFile -Encoding UTF8 -NoNewline + +# Summary +Write-Message "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -Level Info +Write-Message "Matrix Generation Summary" -Level Success +Write-Message "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -Level Info +Write-Message "" +Write-Message "Total Jobs: $($allEntries.Count)" -Level Success +Write-Message "Output File: $outputFile" -Level Success +Write-Message "" +Write-Message "Breakdown by Project:" -Level Info +Write-Message "" + +foreach ($proj in $stats.Keys | Sort-Object) { + $s = $stats[$proj] + + if ($s.Mode -eq 'collection') { + $summary = "$($s.Collections) collection(s) + $($s.Uncollected) uncollected" + Write-Message " 📦 $proj (collection mode): $summary" -Level Info + } + else { + $summary = "$($s.Classes) class(es)" + Write-Message " 📄 $proj (class mode): $summary" -Level Info + } +} + +Write-Message "" +Write-Message "Matrix generation complete! ✨" -Level Success + +#endregion +``` + +## Key Features + +### 1. Dual Mode Support + +```powershell +if ($parsed.Mode -eq 'collection') { + # Collection-based splitting + # Generate: collection entries + uncollected entry +} +else { + # Class-based splitting + # Generate: one entry per class +} +``` + +### 2. Auto-Detection via File Parsing + +```powershell +# Parse .tests.list file format +if ($line -match '^collection:(.+)$') { + $result.Mode = 'collection' + # ... +} +elseif ($line -match '^class:(.+)$') { + $result.Mode = 'class' + # ... +} +``` + +### 3. Unified Matrix Entry Creation + +Each mode has its own entry creator: +- `New-CollectionMatrixEntry`: For collection jobs +- `New-UncollectedMatrixEntry`: For uncollected catch-all +- `New-ClassMatrixEntry`: For individual test classes + +### 4. Rich Logging + +``` +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Processing: Aspire.Hosting.Tests +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Mode: collection + Strategy: Collection-based splitting + + ➕ Collection: DatabaseTests + ➕ Collection: ContainerTests + ➕ Uncollected tests (all non-collection tests) + + ✅ Generated 3 job(s): 2 collection(s) + 1 uncollected +``` + +## Testing the Script + +### Test 1: Collection Mode + +Create test files: + +```bash +# artifacts/helix/Aspire.Hosting.Tests.tests.list +collection:DatabaseTests +collection:ContainerTests +uncollected:* +``` + +```json +// artifacts/helix/Aspire.Hosting.Tests.tests.metadata.json +{ + "projectName": "Aspire.Hosting.Tests", + "testProjectPath": "tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj", + "mode": "collection", + "collections": "DatabaseTests;ContainerTests", + "testSessionTimeout": "25m", + "testHangTimeout": "12m", + "uncollectedTestsSessionTimeout": "15m", + "uncollectedTestsHangTimeout": "8m" +} +``` + +Run script: + +```powershell +pwsh eng/scripts/generate-test-matrix.ps1 ` + -TestListsDirectory ./artifacts/helix ` + -OutputDirectory ./artifacts/test-matrices ` + -BuildOs linux +``` + +**Expected Console Output**: +``` +✅ Starting matrix generation for BuildOs=linux +ℹ️ Test lists directory: ./artifacts/helix +ℹ️ Output directory: ./artifacts/test-matrices + +✅ Found 1 test list file(s) + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +ℹ️ Processing: Aspire.Hosting.Tests +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +ℹ️ Mode: collection +✅ Strategy: Collection-based splitting + +🔍 ➕ Collection: DatabaseTests +🔍 ➕ Collection: ContainerTests +🔍 ➕ Uncollected tests (all non-collection tests) + +✅ Generated 3 job(s): 2 collection(s) + 1 uncollected + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +✅ Matrix Generation Summary +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +✅ Total Jobs: 3 +✅ Output File: ./artifacts/test-matrices/split-tests-matrix.json + +ℹ️ Breakdown by Project: + +ℹ️ 📦 Aspire.Hosting.Tests (collection mode): 2 collection(s) + 1 uncollected + +✅ Matrix generation complete! ✨ +``` + +**Expected JSON Output**: + +```json +{ + "include": [ + { + "type": "collection", + "name": "DatabaseTests", + "shortname": "Collection_DatabaseTests", + "projectName": "Aspire.Hosting.Tests", + "testProjectPath": "tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj", + "filterArg": "--filter-collection \"DatabaseTests\"", + "requiresNugets": false, + "requiresTestSdk": false, + "testSessionTimeout": "25m", + "testHangTimeout": "12m", + "enablePlaywrightInstall": false + }, + { + "type": "collection", + "name": "ContainerTests", + "shortname": "Collection_ContainerTests", + "projectName": "Aspire.Hosting.Tests", + "testProjectPath": "tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj", + "filterArg": "--filter-collection \"ContainerTests\"", + "requiresNugets": false, + "requiresTestSdk": false, + "testSessionTimeout": "25m", + "testHangTimeout": "12m", + "enablePlaywrightInstall": false + }, + { + "type": "uncollected", + "name": "UncollectedTests", + "shortname": "Uncollected", + "projectName": "Aspire.Hosting.Tests", + "testProjectPath": "tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj", + "filterArg": "--filter-not-collection \"DatabaseTests\" --filter-not-collection \"ContainerTests\"", + "requiresNugets": false, + "requiresTestSdk": false, + "testSessionTimeout": "15m", + "testHangTimeout": "8m", + "enablePlaywrightInstall": false + } + ] +} +``` + +### Test 2: Class Mode + +Create test files: + +```bash +# artifacts/helix/Aspire.Templates.Tests.tests.list +class:Aspire.Templates.Tests.BuildAndRunTemplateTests +class:Aspire.Templates.Tests.EmptyTemplateRunTests +class:Aspire.Templates.Tests.StarterTemplateRunTests +``` + +```json +// artifacts/helix/Aspire.Templates.Tests.tests.metadata.json +{ + "projectName": "Aspire.Templates.Tests", + "testClassNamesPrefix": "Aspire.Templates.Tests", + "testProjectPath": "tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj", + "mode": "class", + "collections": "", + "testSessionTimeout": "20m", + "testHangTimeout": "10m", + "requiresNugets": "true", + "requiresTestSdk": "true", + "enablePlaywrightInstall": "true" +} +``` + +Run script: + +```powershell +pwsh eng/scripts/generate-test-matrix.ps1 ` + -TestListsDirectory ./artifacts/helix ` + -OutputDirectory ./artifacts/test-matrices ` + -BuildOs linux +``` + +**Expected Console Output**: +``` +✅ Starting matrix generation for BuildOs=linux +... + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +ℹ️ Processing: Aspire.Templates.Tests +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +ℹ️ Mode: class +✅ Strategy: Class-based splitting + +🔍 ➕ Class: BuildAndRunTemplateTests +🔍 ➕ Class: EmptyTemplateRunTests +🔍 ➕ Class: StarterTemplateRunTests + +✅ Generated 3 job(s): one per class + +... + +ℹ️ 📄 Aspire.Templates.Tests (class mode): 3 class(es) + +✅ Matrix generation complete! ✨ +``` + +**Expected JSON Output**: + +```json +{ + "include": [ + { + "type": "class", + "fullClassName": "Aspire.Templates.Tests.BuildAndRunTemplateTests", + "shortname": "BuildAndRunTemplateTests", + "projectName": "Aspire.Templates.Tests", + "testProjectPath": "tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj", + "filterArg": "--filter-class \"Aspire.Templates.Tests.BuildAndRunTemplateTests\"", + "requiresNugets": true, + "requiresTestSdk": true, + "testSessionTimeout": "20m", + "testHangTimeout": "10m", + "enablePlaywrightInstall": true + }, + { + "type": "class", + "fullClassName": "Aspire.Templates.Tests.EmptyTemplateRunTests", + "shortname": "EmptyTemplateRunTests", + ... + }, + { + "type": "class", + "fullClassName": "Aspire.Templates.Tests.StarterTemplateRunTests", + "shortname": "StarterTemplateRunTests", + ... + } + ] +} +``` + +### Test 3: Mixed Projects + +Create files for both projects above, then run: + +```powershell +pwsh eng/scripts/generate-test-matrix.ps1 ` + -TestListsDirectory ./artifacts/helix ` + -OutputDirectory ./artifacts/test-matrices ` + -BuildOs linux +``` + +**Expected**: 6 total jobs (3 from Hosting.Tests + 3 from Templates.Tests) + +**Console Summary**: +``` +ℹ️ Breakdown by Project: + +ℹ️ 📦 Aspire.Hosting.Tests (collection mode): 2 collection(s) + 1 uncollected +ℹ️ 📄 Aspire.Templates.Tests (class mode): 3 class(es) +``` + +## Validation + +### Verify Matrix Structure + +```powershell +# Load matrix +$matrix = Get-Content ./artifacts/test-matrices/split-tests-matrix.json | ConvertFrom-Json + +# Check entry count +$matrix.include.Count + +# Verify all entries have required fields +$matrix.include | ForEach-Object { + $required = @('type', 'shortname', 'projectName', 'testProjectPath', 'filterArg') + foreach ($field in $required) { + if (-not $_.$field) { + Write-Error "Missing field: $field in entry: $($_.shortname)" + } + } +} + +# Check filter arguments +$matrix.include | Select-Object shortname, filterArg | Format-Table + +# Group by type +$matrix.include | Group-Object -Property type | Select-Object Name, Count +``` + +### Verify Filter Arguments Work + +```powershell +# Test a collection filter +dotnet test YourTests.dll -- --filter-collection "DatabaseTests" + +# Test a class filter +dotnet test YourTests.dll -- --filter-class "Aspire.Templates.Tests.Test1" + +# Test uncollected filter +dotnet test YourTests.dll -- --filter-not-collection "DatabaseTests" --filter-not-collection "ContainerTests" +``` + +## Common Issues + +### Issue 1: "Mode is unknown" + +**Symptom**: Script skips project with "Unable to determine mode" +**Cause**: .tests.list file has unexpected format +**Fix**: Check file format - should have `collection:` or `class:` prefixes + +### Issue 2: Invalid JSON + +**Symptom**: GitHub Actions can't parse matrix +**Cause**: Special characters in names +**Fix**: Script escapes quotes automatically, but verify with `jq` + +```bash +cat split-tests-matrix.json | jq empty +# Should exit with code 0 if valid +``` + +### Issue 3: Empty filterArg for uncollected + +**Symptom**: Uncollected job has empty filter +**Cause**: No collections to exclude +**Fix**: This is OK - empty filter runs all tests + +## Next Steps + +The matrix is now generated! GitHub Actions workflow already consumes it (no changes needed from v1). + +Proceed to [Step 4: Project Configuration (v3)](./STEP_04_PROJECT_CONFIG_V3.md) \ No newline at end of file diff --git a/docs/test-splitting/STEP_04_PROJECT_CONFIG.md b/docs/test-splitting/STEP_04_PROJECT_CONFIG.md new file mode 100644 index 00000000000..3cb4508aac8 --- /dev/null +++ b/docs/test-splitting/STEP_04_PROJECT_CONFIG.md @@ -0,0 +1,230 @@ +# Step 4: Project Configuration + +## Overview + +Configure test projects to use the new unified splitting mechanism. This step shows how to migrate existing projects and enable new ones. + +## Configuration Properties + +### Required Properties (for splitting) + +```xml + +true + + +Aspire.Hosting.Tests +``` + +### Optional Properties + +```xml + +QuickTest1;QuickTest2 + + +25m +12m + + +true + + +true + + +true +``` + +## Migration: Aspire.Templates.Tests + +### Before (Custom Implementation) + +```xml + + + $(DefaultTargetFramework) + + true + true + + xunit.runner.json + $(TestArchiveTestsDirForTemplateTests) + + + true + Aspire.Templates.Tests + + $(NoWarn);xUnit1051 + true + + + + + + + + + + + + +``` + +### After (Unified Mechanism) + +```xml + + + $(DefaultTargetFramework) + + true + true + + xunit.runner.json + + + true + Aspire.Templates.Tests + + + true + true + true + + + 20m + 12m + + $(NoWarn);xUnit1051 + true + + + + + + + + + + + + +``` + +### Changes Summary + +- ✅ Replace `ExtractTestClassNamesForHelix` with `SplitTestsForCI` +- ✅ Keep `TestClassNamesPrefix` (same property name) +- ✅ Add `RequiresNugetsForSplitTests=true` +- ✅ Add `RequiresTestSdkForSplitTests=true` +- ✅ Add `EnablePlaywrightInstallForSplitTests=true` +- ✅ Add timeout configurations +- ✅ Remove `TestArchiveTestsDir` override (use default) + +## New Project: Aspire.Hosting.Tests + +### Complete Configuration + +```xml + + + $(DefaultTargetFramework) + + + true + Aspire.Hosting.Tests + + + 25m + 15m + + + false + false + false + + + + + + + +``` + +## OS-Specific Opt-In/Out + +### Example: Linux-Only Splitting + +Some projects may only need splitting on Linux (e.g., Docker tests): + +```xml + + + true + Aspire.Docker.Tests + + + true + false + true + +``` + +This creates: +- **Linux**: Split into multiple jobs (one per class) +- **Windows**: Single job (no splitting) +- **macOS**: Doesn't run at all + +## Projects to Enable Splitting + +### High Priority (Long-Running) + +1. **Aspire.Templates.Tests** ✅ (Already has splitting, migrate to new mechanism) + - Currently: ~15 test classes + - Timeout: 20m + - Needs: Packages, SDK, Playwright + +2. **Aspire.Hosting.Tests** 🎯 (Primary target) + - Estimated: 50+ test classes + - Timeout: 25m + - Needs: None (regular integration test) + +3. **Aspire.Hosting.*.Tests** (if long-running) + - Aspire.Hosting.Azure.Tests + - Aspire.Hosting.Postgres.Tests + - etc. + +### Medium Priority + +4. Other integration tests if they exceed 15 minutes + +### Low Priority + +- Unit tests (usually fast enough) +- Tests with < 5 test classes (overhead not worth it) + +## Configuration Decision Tree + +``` +Is the test project slow (>15 minutes)? +│ +├─ NO → Don't enable splitting +│ (Keep as regular test) +│ +└─ YES → Does it have >5 test classes? + │ + ├─ NO → Don't enable splitting + │ (Won't benefit from parallelization) + │ + └─ YES → Enable splitting! + │ + ├─ Set SplitTestsForCI=true + ├─ Set TestClassNamesPrefix + ├─ Set custom timeouts if needed + └─ Set requirements (packages/SDK/etc.) +``` + +## Validation Checklist + +Before \ No newline at end of file diff --git a/docs/test-splitting/STEP_04_PROJECT_CONFIG_V2.md b/docs/test-splitting/STEP_04_PROJECT_CONFIG_V2.md new file mode 100644 index 00000000000..9e87498a599 --- /dev/null +++ b/docs/test-splitting/STEP_04_PROJECT_CONFIG_V2.md @@ -0,0 +1,490 @@ +# Step 4: Project Configuration (v2 - Collection Support) + +## Overview + +Configure test projects to use collection-based splitting with examples showing how to optimize test execution. + +## Configuration Properties + +### Required Properties + +```xml + +true + + +Aspire.Hosting.Tests +``` + +### Optional Properties (v2 Enhancements) + +```xml + +QuickTests;FastTests + + +25m +12m + + +15m +8m + + +false +false +false +``` + +## Example 1: Aspire.Hosting.Tests (NEW - Collections) + +### Project File Configuration + +```xml + + + $(DefaultTargetFramework) + + + true + Aspire.Hosting.Tests + + + 30m + 15m + + + 15m + 8m + + + false + false + + + + +``` + +### Test Class Organization + +```csharp +using Xunit; + +namespace Aspire.Hosting.Tests; + +// Slow database tests - group together +[Collection("DatabaseIntegration")] +public class PostgresLifecycleTests +{ + [Fact] + public async Task CanStartPostgresContainer() + { + // 2-3 minutes per test + } + + [Fact] + public async Task CanConnectToPostgres() + { + // 2-3 minutes per test + } +} + +[Collection("DatabaseIntegration")] +public class SqlServerLifecycleTests +{ + [Fact] + public async Task CanStartSqlServerContainer() + { + // 2-3 minutes per test + } +} + +// Slow container tests - separate group +[Collection("ContainerLifecycle")] +public class DockerContainerTests +{ + [Fact] + public async Task CanStartGenericContainer() + { + // 2-3 minutes per test + } + + [Fact] + public async Task CanStopContainer() + { + // 2 minutes per test + } +} + +[Collection("ContainerLifecycle")] +public class ContainerNetworkingTests +{ + [Fact] + public async Task ContainersCanCommunicate() + { + // 3 minutes per test + } +} + +// Fast unit tests - NO collection attribute +public class ConfigurationTests +{ + [Fact] + public void CanParseConfiguration() + { + // < 1 second + } + + [Fact] + public void CanValidateSettings() + { + // < 1 second + } +} + +public class UtilityTests +{ + [Fact] + public void HelperMethodWorks() + { + // < 1 second + } +} +``` + +### Expected CI Behavior + +**Before** (1 job): +``` +Aspire.Hosting.Tests: 55 minutes +``` + +**After** (3 jobs running in parallel): +``` +Collection_DatabaseIntegration: ~20 minutes (Postgres + SqlServer tests) +Collection_ContainerLifecycle: ~15 minutes (Docker + Networking tests) +UncollectedTests: ~5 minutes (Config + Utility tests) +``` + +**Total CI Time**: ~20 minutes (60% reduction!) + +## Example 2: Aspire.Templates.Tests (MIGRATED) + +### Before (v1 - Class-based splitting) + +```xml + + + $(DefaultTargetFramework) + + + true + Aspire.Templates.Tests + + true + true + + +``` + +### After (v2 - Collection-based splitting) + +```xml + + + $(DefaultTargetFramework) + + + true + Aspire.Templates.Tests + + + true + true + true + + + 25m + 15m + + + 15m + 10m + + true + true + + +``` + +### Test Class Organization Strategy + +```csharp +using Xunit; + +namespace Aspire.Templates.Tests; + +// Slow Playwright tests for starter template - group together +[Collection("StarterTemplateWithPlaywright")] +public class StarterTemplateProjectNamesTests +{ + // Each test: 3-5 minutes (Playwright browser automation) +} + +[Collection("StarterTemplateWithPlaywright")] +public class StarterTemplateRunTests +{ + // Each test: 3-5 minutes +} + +// Slow Playwright tests for basic template - separate group +[Collection("BasicTemplateWithPlaywright")] +public class BuildAndRunTemplateTests +{ + // Each test: 3-5 minutes +} + +// Build-only tests (no Playwright) - NO collection +public class NewUpAndBuildStandaloneTemplateTests +{ + // Each test: 1-2 minutes (just dotnet build) +} + +public class TemplateManifestTests +{ + // Each test: < 1 minute (metadata tests) +} +``` + +**Result**: 3 jobs +1. Collection_StarterTemplateWithPlaywright (~15 min) +2. Collection_BasicTemplateWithPlaywright (~12 min) +3. UncollectedTests (~5 min) + +## Example 3: Simple Project (No Collections) + +### When NOT to Use Collections + +```xml + + + $(DefaultTargetFramework) + + + true + Aspire.MySqlConnector.Tests + + + 15m + + +``` + +```csharp +// All test classes without [Collection] attribute +public class ConnectionTests { } +public class QueryTests { } +public class TransactionTests { } +``` + +**Result**: 1 job (UncollectedTests) running all tests + +**When to use this**: +- Project has < 15 minute total runtime +- All tests are similar speed +- No benefit from parallelization + +## Example 4: Excluding Collections + +### Scenario: Some Collections Shouldn't Split + +```xml + + true + Aspire.Hosting.Tests + + + QuickIntegrationTests;FastSmokeTests + +``` + +```csharp +[Collection("SlowDatabaseTests")] +public class SlowTests { } // Gets own job + +[Collection("QuickIntegrationTests")] // Excluded from splitting +public class QuickTests { } // Runs in UncollectedTests + +public class OtherTests { } // Runs in UncollectedTests +``` + +**Result**: 2 jobs +1. Collection_SlowDatabaseTests +2. UncollectedTests (includes QuickIntegrationTests + OtherTests) + +## Decision Matrix: Should You Use Collections? + +### ✅ Use Collections When: + +| Scenario | Example | +|----------|---------| +| **Shared expensive setup** | Database containers that multiple test classes use | +| **Long-running integration tests** | Tests that take 2+ minutes each | +| **Logical test grouping** | All Azure tests, all Docker tests, etc. | +| **Similar resource needs** | Tests that all need Playwright, or all need databases | + +### ❌ Don't Use Collections When: + +| Scenario | Reason | +|----------|--------| +| **Fast unit tests** | Overhead isn't worth it; let them run together | +| **< 5 total test classes** | Not enough parallelization benefit | +| **Tests need isolation** | Collections share fixtures which may cause conflicts | +| **Total runtime < 15 min** | Single job is fast enough | + +## Migration Checklist + +### For Each Long-Running Project: + +- [ ] Analyze test suite duration +- [ ] Identify slow test groups (> 10 min combined) +- [ ] Add `[Collection("GroupName")]` to slow test classes +- [ ] Keep fast tests without collection attribute +- [ ] Update .csproj with split configuration +- [ ] Set appropriate timeouts +- [ ] Test locally first +- [ ] Monitor CI times after merge + +## Best Practices + +### 1. Collection Naming + +```csharp +// ✅ Good: Descriptive, indicates purpose +[Collection("DatabaseIntegrationTests")] +[Collection("ContainerLifecycleTests")] +[Collection("PlaywrightAutomationTests")] + +// ❌ Bad: Too vague or too specific +[Collection("Tests")] // Too vague +[Collection("PostgresTest1")] // Too specific +``` + +### 2. Collection Size + +```csharp +// ✅ Good: Multiple related test classes in one collection +[Collection("DatabaseTests")] +public class PostgresTests { /* 10 tests */ } + +[Collection("DatabaseTests")] +public class MySqlTests { /* 8 tests */ } + +[Collection("DatabaseTests")] +public class SqlServerTests { /* 12 tests */ } +// Total: 30 tests, ~20 minutes - good parallelization unit + +// ❌ Bad: One test class per collection +[Collection("PostgresTests")] +public class PostgresTests { /* 10 tests */ } + +[Collection("MySqlTests")] +public class MySqlTests { /* 8 tests */ } +// Too granular, overhead not worth it +``` + +### 3. Timeout Configuration + +```xml + +30m + + +10m +``` + +### 4. Test Isolation + +```csharp +// ✅ Good: Tests in same collection can share fixtures +[Collection("DatabaseTests")] +public class PostgresTests : IClassFixture +{ + // Fixture is shared across collection +} + +[Collection("DatabaseTests")] +public class MySqlTests : IClassFixture +{ + // Same fixture instance - efficient! +} + +// ❌ Bad: Tests that MUST be isolated shouldn't share collection +[Collection("IsolatedTests")] // Don't do this +public class Test1 { /* Modifies global state */ } + +[Collection("IsolatedTests")] // Will conflict with Test1 +public class Test2 { /* Also modifies global state */ } +``` + +## Validation After Configuration + +### 1. Build Locally + +```bash +dotnet build tests/YourProject.Tests/YourProject.Tests.csproj \ + /t:Build;ExtractTestClassNames \ + -p:PrepareForHelix=true \ + -p:SplitTestsForCI=true \ + -p:TestClassNamesPrefix=YourProject.Tests +``` + +### 2. Check Generated Files + +```bash +# Should see: +ls artifacts/helix/YourProject.Tests.tests.list +ls artifacts/helix/YourProject.Tests.tests.metadata.json + +# Content should be: +cat artifacts/helix/YourProject.Tests.tests.list +# collection:YourCollection1 +# collection:YourCollection2 +# uncollected:* +``` + +### 3. Generate Matrix + +```bash +pwsh eng/scripts/generate-test-matrix.ps1 \ + -TestListsDirectory ./artifacts/helix \ + -OutputDirectory ./artifacts/test-matrices \ + -BuildOs linux +``` + +### 4. Verify Matrix + +```bash +cat artifacts/test-matrices/split-tests-matrix.json | jq '.include[] | {name, filterArg}' +``` + +**Expected output**: +```json +{ + "name": "YourCollection1", + "filterArg": "--filter-collection \"YourCollection1\"" +} +{ + "name": "YourCollection2", + "filterArg": "--filter-collection \"YourCollection2\"" +} +{ + "name": "UncollectedTests", + "filterArg": "--filter-not-collection \"YourCollection1\" --filter-not-collection \"YourCollection2\"" +} +``` + +## Next Steps + +Proceed to [Step 5: Testing & Validation (v2)](./STEP_05_TESTING_V2.md) \ No newline at end of file diff --git a/docs/test-splitting/STEP_04_PROJECT_CONFIG_V3.md b/docs/test-splitting/STEP_04_PROJECT_CONFIG_V3.md new file mode 100644 index 00000000000..f1093378513 --- /dev/null +++ b/docs/test-splitting/STEP_04_PROJECT_CONFIG_V3.md @@ -0,0 +1,316 @@ +# Step 4: Project Configuration (v3 - Simplified) + +## Overview + +With v3's auto-detection, project configuration is minimal. Just set two properties and the system automatically detects whether to use collection or class-based splitting. + +## Minimal Configuration + +### Required Properties (Only 2!) + +```xml + + + true + + + YourProject.Tests + +``` + +That's it! The system auto-detects collections and chooses the optimal strategy. + +## Configuration Examples + +### Example 1: Aspire.Hosting.Tests (NEW - With Collections) + +#### Step 1: Configure Project + +```xml name=tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj + + + $(DefaultTargetFramework) + + + true + Aspire.Hosting.Tests + + + 30m + 15m + 15m + 8m + + + + +``` + +#### Step 2: Add Collections to Test Classes + +```csharp +using Xunit; + +namespace Aspire.Hosting.Tests; + +// Group slow database tests together +[Collection("DatabaseIntegration")] +public class PostgresLifecycleTests +{ + [Fact] + public async Task CanStartPostgresContainer() + { + // Test implementation + } +} + +[Collection("DatabaseIntegration")] +public class MySqlLifecycleTests +{ + [Fact] + public async Task CanStartMySqlContainer() + { + // Test implementation + } +} + +// Group container tests together +[Collection("ContainerLifecycle")] +public class DockerContainerTests +{ + [Fact] + public async Task CanStartGenericContainer() + { + // Test implementation + } +} + +// Fast tests - NO collection attribute +public class ConfigurationTests +{ + [Fact] + public void CanParseConfiguration() + { + // Fast unit test + } +} + +public class UtilityTests +{ + [Fact] + public void HelperMethodWorks() + { + // Fast unit test + } +} +``` + +#### Result + +**Auto-detected mode**: Collection (2 collections found) +**CI Jobs**: 3 +- `Collection_DatabaseIntegration` (Postgres + MySQL tests) +- `Collection_ContainerLifecycle` (Docker tests) +- `Uncollected` (Configuration + Utility tests) + +**Before**: 1 job, 60 minutes +**After**: 3 parallel jobs, ~25 minutes (58% reduction) + +### Example 2: Aspire.Templates.Tests (MIGRATE from Old System) + +#### Before (Custom Mechanism) + +```xml + + + true + Aspire.Templates.Tests + $(TestArchiveTestsDirForTemplateTests) + +``` + +#### After (Unified v3 Mechanism) + +```xml name=tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj + + + $(DefaultTargetFramework) + + + true + Aspire.Templates.Tests + + + true + true + true + + + 20m + 12m + + + true + true + xunit.runner.json + $(NoWarn);xUnit1051 + + + + + + + + + + + +``` + +#### Test Classes (No Changes Needed) + +```csharp +// Existing test classes without [Collection] attributes +public class BuildAndRunTemplateTests { } +public class EmptyTemplateRunTests { } +public class StarterTemplateRunTests { } +// ... etc +``` + +#### Result + +**Auto-detected mode**: Class (no collections found) +**CI Jobs**: 12 (one per test class) +**Behavior**: Identical to old system, but using unified infrastructure + +### Example 3: Simple Project (No Splitting Needed) + +```xml name=tests/Aspire.MySqlConnector.Tests/Aspire.MySqlConnector.Tests.csproj + + + $(DefaultTargetFramework) + +``` + +**Result**: 1 job (existing behavior, no splitting) + +## Optional Configuration Properties + +### Timeouts + +```xml + +20m +10m + + +15m +8m +``` + +### Test Requirements + +```xml + +true + + +true + + +true +``` + +### Collection Management + +```xml + +FastTests;QuickTests +``` + +These collections will run in the `Uncollected` job instead. + +## Decision Guide + +### Should I Enable Splitting? + +``` +Is total test time > 15 minutes? +│ +├─ NO → Don't enable SplitTestsOnCI +│ Overhead not worth it +│ +└─ YES → Enable SplitTestsOnCI=true + │ + Do you have logical test groups? + │ + ├─ YES → Add [Collection] attributes + │ System auto-detects: Collection mode + │ Result: Fewer jobs, better parallelization + │ + └─ NO → Leave tests as-is + System auto-detects: Class mode + Result: One job per class +``` + +### Collection Size Guidelines + +**Good Collection** (15-30 minutes): +```csharp +[Collection("DatabaseTests")] +public class PostgresTests { /* 20 tests, 8 min */ } + +[Collection("DatabaseTests")] +public class MySqlTests { /* 15 tests, 7 min */ } + +[Collection("DatabaseTests")] +public class SqlServerTests { /* 25 tests, 10 min */ } + +// Total: ~25 minutes - ideal for one job +``` + +**Too Small** (< 5 minutes): +```csharp +[Collection("QuickTest")] +public class OneTest { /* 2 tests, 1 min */ } + +// Don't create collections for fast tests +// Let them run in the uncollected job +``` + +**Too Large** (> 45 minutes): +```csharp +[Collection("AllDatabaseTests")] +public class Test1 { /* 100 tests */ } +public class Test2 { /* 100 tests */ } +// ... + +// Split into multiple smaller collections instead +``` + +## Migration Checklist + +### For Each Long-Running Project: + +- [ ] Measure current test duration +- [ ] If > 15 min, enable `SplitTestsOnCI=true` +- [ ] Set `TestClassNamesPrefix` +- [ ] (Optional) Add `[Collection]` to slow test groups +- [ ] Test locally (see Step 5) +- [ ] Create PR +- [ ] Monitor CI times after merge + +### Specific Migration: Aspire.Templates.Tests + +- [ ] Replace `ExtractTestClassNamesForHelix` with `SplitTestsOnCI` +- [ ] Keep `TestClassNamesPrefix` (same name) +- [ ] Add `RequiresNugetsForSplitTests=true` +- [ ] Add `RequiresTestSdkForSplitTests=true` +- [ ] Add `EnablePlaywrightInstallForSplitTests=true` +- [ ] Remove `TestArchiveTestsDir` override +- [ ] Test locally +- [ ] Verify same number of jobs in CI + +## Next Steps + +Proceed to [Step 5: Testing & Validation](./STEP_05_TESTING_V3.md) \ No newline at end of file diff --git a/docs/test-splitting/STEP_05_TESTING_V3.md b/docs/test-splitting/STEP_05_TESTING_V3.md new file mode 100644 index 00000000000..92b3e54e3a0 --- /dev/null +++ b/docs/test-splitting/STEP_05_TESTING_V3.md @@ -0,0 +1,373 @@ +# Step 5: Testing & Validation Guide + +## Overview + +This guide provides step-by-step instructions for testing the implementation locally before pushing to CI. + +## Prerequisites + +- PowerShell 7.0+ installed +- .NET SDK matching `global.json` +- Aspire repository cloned locally + +## Phase 1: Test PowerShell Scripts in Isolation + +### Test 1: Discovery Helper Script + +```powershell +# Create mock test output +$mockOutput = @( + "Collection: DatabaseTests", + " Aspire.Hosting.Tests.PostgresTests.CanStartContainer", + " Aspire.Hosting.Tests.PostgresTests.CanConnect", + "Collection: ContainerTests", + " Aspire.Hosting.Tests.DockerTests.CanStartContainer", + "Aspire.Hosting.Tests.QuickTests.FastTest1" +) + +# Test the script +pwsh eng/scripts/extract-test-metadata.ps1 ` + -TestAssemblyOutput $mockOutput ` + -TestClassNamesPrefix "Aspire.Hosting.Tests" ` + -OutputListFile "./test-output.list" +``` + +**Expected Output File**: +``` +collection:ContainerTests +collection:DatabaseTests +uncollected:* +``` + +**Validation**: +- [ ] Script runs without errors +- [ ] Output file created +- [ ] Contains 3 lines (2 collections + uncollected) +- [ ] Collections are sorted alphabetically + +### Test 2: Matrix Generator Script + +```powershell +# Create test files +mkdir -p artifacts/helix + +# Create .tests.list +@" +collection:DatabaseTests +collection:ContainerTests +uncollected:* +"@ | Out-File -FilePath artifacts/helix/TestProject.tests.list -Encoding UTF8 + +# Create .tests.metadata.json +@" +{ + "projectName": "TestProject", + "testProjectPath": "tests/TestProject/TestProject.csproj", + "mode": "collection", + "collections": "DatabaseTests;ContainerTests", + "testSessionTimeout": "20m", + "testHangTimeout": "10m" +} +"@ | Out-File -FilePath artifacts/helix/TestProject.tests.metadata.json -Encoding UTF8 + +# Run generator +pwsh eng/scripts/generate-test-matrix.ps1 ` + -TestListsDirectory ./artifacts/helix ` + -OutputDirectory ./artifacts/test-matrices ` + -BuildOs linux +``` + +**Expected Output**: +- [ ] Matrix JSON file created +- [ ] Contains 3 entries (2 collections + 1 uncollected) +- [ ] Each entry has `type`, `name`, `shortname`, `filterArg`, etc. +- [ ] Filter args are correct: + - `--filter-collection "DatabaseTests"` + - `--filter-collection "ContainerTests"` + - `--filter-not-collection "DatabaseTests" --filter-not-collection "ContainerTests"` + +**Validate JSON**: +```powershell +# Check JSON is valid +$matrix = Get-Content ./artifacts/test-matrices/split-tests-matrix.json | ConvertFrom-Json +$matrix.include.Count # Should be 3 + +# Or use jq +jq '.include | length' ./artifacts/test-matrices/split-tests-matrix.json +# Should output: 3 +``` + +## Phase 2: Test MSBuild Integration + +### Test 1: Build Test Project with Splitting Enabled + +Choose a test project to experiment with (or create a dummy one): + +```bash +# Build with splitting enabled +dotnet build tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj \ + /t:Build;ExtractTestClassNames \ + -p:PrepareForHelix=true \ + -p:SplitTestsOnCI=true \ + -p:TestClassNamesPrefix=Aspire.Templates.Tests \ + -p:InstallBrowsersForPlaywright=false \ + /bl:build.binlog +``` + +**Expected Output**: +``` +[Aspire.Templates.Tests] Starting test metadata extraction... +[Aspire.Templates.Tests] Running discovery helper... +ℹ️ Parsing test assembly output... +✅ Detection Results: +ℹ️ Mode: class (or "collection" if you added [Collection] attributes) +... +[Aspire.Templates.Tests] ✅ Test metadata extraction complete! +``` + +**Validation**: +- [ ] Build succeeds +- [ ] Files created in `artifacts/helix/`: + - [ ] `Aspire.Templates.Tests.tests.list` + - [ ] `Aspire.Templates.Tests.tests.metadata.json` +- [ ] Binlog shows ExtractTestClassNames target executed +- [ ] No errors in console output + +### Test 2: Verify Generated Files + +```bash +# Check .tests.list +cat artifacts/helix/Aspire.Templates.Tests.tests.list + +# Check metadata +cat artifacts/helix/Aspire.Templates.Tests.tests.metadata.json | jq . + +# Verify mode +cat artifacts/helix/Aspire.Templates.Tests.tests.metadata.json | jq -r .mode +# Should output: "class" or "collection" +``` + +### Test 3: Generate Matrix + +```bash +# Run the full GetTestProjects.proj +dotnet build tests/Shared/GetTestProjects.proj \ + /p:TestsListOutputPath=$PWD/artifacts/TestsForGithubActions.list \ + /p:TestMatrixOutputPath=$PWD/artifacts/test-matrices/ \ + /p:ContinuousIntegrationBuild=true \ + /bl:get-test-projects.binlog +``` + +**Validation**: +- [ ] `artifacts/TestsForGithubActions.list` created (regular tests) +- [ ] `artifacts/TestsForGithubActions.list.split-projects` created (split tests) +- [ ] `artifacts/test-matrices/split-tests-matrix.json` created +- [ ] Matrix JSON is valid + +```bash +# Validate +jq . artifacts/test-matrices/split-tests-matrix.json +``` + +## Phase 3: Test with Real Project + +### Option A: Test with Aspire.Templates.Tests (No Collections) + +```bash +# 1. Update .csproj (already has splitting, just verify) +# 2. Build +./build.sh -restore -build -projects tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj + +# 3. Extract metadata +dotnet build tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj \ + /t:Build;ExtractTestClassNames \ + -p:PrepareForHelix=true \ + -p:SplitTestsOnCI=true \ + -p:InstallBrowsersForPlaywright=false + +# 4. Check mode +cat artifacts/helix/Aspire.Templates.Tests.tests.metadata.json | jq -r .mode +# Expected: "class" + +# 5. Count entries +cat artifacts/helix/Aspire.Templates.Tests.tests.list | wc -l +# Expected: ~12 (one per test class) +``` + +### Option B: Test with Aspire.Hosting.Tests (Add Collections) + +```bash +# 1. Add [Collection] attributes to some test classes +# Edit: tests/Aspire.Hosting.Tests/SomeTests.cs + +# 2. Enable splitting in .csproj +# Add: +# true +# Aspire.Hosting.Tests + +# 3. Build +./build.sh -restore -build -projects tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj + +# 4. Extract metadata +dotnet build tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj \ + /t:Build;ExtractTestClassNames \ + -p:PrepareForHelix=true \ + -p:SplitTestsOnCI=true + +# 5. Check mode +cat artifacts/helix/Aspire.Hosting.Tests.tests.metadata.json | jq -r .mode +# Expected: "collection" + +# 6. Check collections +cat artifacts/helix/Aspire.Hosting.Tests.tests.list +# Expected: +# collection:YourCollectionName +# collection:AnotherCollection +# uncollected:* +``` + +## Phase 4: Test Full Workflow Locally + +### Simulate GitHub Actions Enumerate Step + +```bash +# Run the enumerate-tests action logic locally +dotnet build tests/Shared/GetTestProjects.proj \ + /p:TestsListOutputPath=$PWD/artifacts/TestsForGithubActions.list \ + /p:TestMatrixOutputPath=$PWD/artifacts/test-matrices/ \ + /p:ContinuousIntegrationBuild=true + +# Check split projects +cat artifacts/TestsForGithubActions.list.split-projects +# Should list: Templates or Hosting (whichever has SplitTestsOnCI=true) + +# Build each split project +while read project; do + echo "Building $project..." + dotnet build tests/Aspire.$project.Tests/Aspire.$project.Tests.csproj \ + /t:Build;ExtractTestClassNames \ + -p:PrepareForHelix=true \ + -p:SplitTestsOnCI=true \ + -p:InstallBrowsersForPlaywright=false +done < artifacts/TestsForGithubActions.list.split-projects + +# Generate matrix +pwsh eng/scripts/generate-test-matrix.ps1 \ + -TestListsDirectory ./artifacts/helix \ + -OutputDirectory ./artifacts/test-matrices \ + -BuildOs linux + +# Verify matrix +jq '.include[] | {shortname, filterArg}' artifacts/test-matrices/split-tests-matrix.json +``` + +## Phase 5: Verify Filter Arguments Work + +### Test Collection Filter + +```bash +# Run tests with collection filter +dotnet test artifacts/bin/Aspire.Hosting.Tests/Debug/net9.0/Aspire.Hosting.Tests.dll \ + -- --filter-collection "DatabaseTests" + +# Should only run tests in DatabaseTests collection +``` + +### Test Class Filter + +```bash +# Run tests with class filter +dotnet test artifacts/bin/Aspire.Templates.Tests/Debug/net9.0/Aspire.Templates.Tests.dll \ + -- --filter-class "Aspire.Templates.Tests.BuildAndRunTemplateTests" + +# Should only run tests in that class +``` + +### Test Uncollected Filter + +```bash +# Run tests NOT in collections +dotnet test artifacts/bin/Aspire.Hosting.Tests/Debug/net9.0/Aspire.Hosting.Tests.dll \ + -- --filter-not-collection "DatabaseTests" --filter-not-collection "ContainerTests" + +# Should only run tests without [Collection] attributes +``` + +## Validation Checklist + +### PowerShell Scripts +- [ ] `extract-test-metadata.ps1` runs without errors +- [ ] `extract-test-metadata.ps1` detects collections correctly +- [ ] `extract-test-metadata.ps1` falls back to class mode when no collections +- [ ] `generate-test-matrix.ps1` creates valid JSON +- [ ] `generate-test-matrix.ps1` handles both collection and class modes + +### MSBuild Integration +- [ ] ExtractTestClassNames target executes +- [ ] `.tests.list` file is generated +- [ ] `.tests.metadata.json` file is generated +- [ ] Mode is correctly detected and stored in metadata +- [ ] GetTestProjects.proj identifies split projects + +### Generated Artifacts +- [ ] `.tests.list` format is correct +- [ ] `.tests.metadata.json` is valid JSON +- [ ] `split-tests-matrix.json` is valid JSON +- [ ] All matrix entries have required fields +- [ ] Filter arguments have correct syntax + +### xUnit Filters +- [ ] `--filter-collection` works +- [ ] `--filter-class` works +- [ ] `--filter-not-collection` works +- [ ] Filters run expected number of tests + +## Troubleshooting + +### Issue: "PowerShell script not found" + +**Error**: `Cannot find path 'eng/scripts/extract-test-metadata.ps1'` + +**Fix**: Ensure working directory is repository root: +```bash +cd /path/to/aspire +pwd # Should show aspire repo root +``` + +### Issue: "No tests found matching prefix" + +**Error**: `Error: No test classes found matching prefix` + +**Fix**: Verify `TestClassNamesPrefix` matches actual test namespace: +```bash +# Check test namespace +grep -r "^namespace " tests/YourProject.Tests/*.cs | head -1 +# Should match TestClassNamesPrefix +``` + +### Issue: "Mode is empty in metadata" + +**Error**: Mode field is empty or missing + +**Fix**: Check PowerShell script output - may have parsing errors. +Look in binlog for script console output. + +### Issue: "Matrix JSON is invalid" + +**Error**: GitHub Actions can't parse matrix + +**Fix**: Validate JSON locally: +```bash +jq empty artifacts/test-matrices/split-tests-matrix.json +# Exit code 0 = valid, non-zero = invalid +``` + +## Next Steps + +Once local testing passes: +1. Create PR with changes +2. Push to branch +3. Monitor GitHub Actions workflow +4. Verify matrices are generated correctly +5. Verify tests run in split jobs +6. Compare CI times before/after \ No newline at end of file diff --git a/docs/test-splitting/STEP_06_CI_INTEGRATION.md b/docs/test-splitting/STEP_06_CI_INTEGRATION.md new file mode 100644 index 00000000000..4e4664dfad7 --- /dev/null +++ b/docs/test-splitting/STEP_06_CI_INTEGRATION.md @@ -0,0 +1,318 @@ +# Step 6: CI Integration & Verification + +## Overview + +This guide explains how to verify the GitHub Actions integration and what to expect when your PR runs in CI. + +## GitHub Actions Workflow + +The existing `.github/workflows/tests.yml` workflow already supports the new matrix format from v1. No changes are needed because: + +1. The enumerate-tests action outputs `split_tests_matrix` +2. The workflow consumes it with `fromJson()` +3. The run-tests workflow accepts the matrix fields + +### Workflow Flow + +``` +setup_for_tests_lin (ubuntu-latest) + ↓ +enumerate-tests action + ↓ + ├─ Build split test projects + ├─ Call extract-test-metadata.ps1 + ├─ Call generate-test-matrix.ps1 + └─ Output: split_tests_matrix JSON + ↓ +split_tests_lin job + ↓ +Uses matrix: fromJson(needs.setup.outputs.split_tests_matrix) + ↓ +For each matrix entry: + - testShortName: ${{ matrix.shortname }} + - testProjectPath: ${{ matrix.testProjectPath }} + - filterArg: ${{ matrix.filterArg }} + - requiresNugets: ${{ matrix.requiresNugets }} + - etc. +``` + +### Key Matrix Fields Used by Workflow + +The workflow expects these fields (all present in v3 output): + +```yaml +matrix: + shortname: "Collection_DatabaseTests" # Used for job name + projectName: "Aspire.Hosting.Tests" # Used in filterArg + testProjectPath: "tests/..." # Which project to test + filterArg: "--filter-collection ..." # xUnit filter + requiresNugets: true/false # Download packages? + requiresTestSdk: true/false # Need test SDK? + testSessionTimeout: "20m" # Timeout + testHangTimeout: "10m" # Hang timeout + enablePlaywrightInstall: true/false # Install browsers? +``` + +All of these are generated by our scripts, so the workflow "just works". + +## What to Expect in CI + +### Setup Jobs (Per OS) + +**setup_for_tests_lin**, **setup_for_tests_macos**, **setup_for_tests_win** + +Each OS runs independently and generates its own matrix: + +``` +✓ Checkout code +✓ Set up .NET +✓ Generate test project lists + → Runs GetTestProjects.proj +✓ Build split test projects + → For each project in .split-projects + → Runs ExtractTestClassNames target + → Calls extract-test-metadata.ps1 +✓ Load split tests matrix + → Calls generate-test-matrix.ps1 + → Outputs JSON to GITHUB_OUTPUT +✓ Upload artifacts (binlogs, lists, matrices) +``` + +**Expected Duration**: 5-10 minutes per OS + +### Split Test Jobs + +**split_tests_lin**, **split_tests_macos**, **split_tests_win** + +If your project has splitting enabled, you'll see new jobs appear: + +**Collection Mode Example**: +``` +split_tests_lin / Aspire.Hosting.Tests_Collection_DatabaseTests (ubuntu-latest) +split_tests_lin / Aspire.Hosting.Tests_Collection_ContainerTests (ubuntu-latest) +split_tests_lin / Aspire.Hosting.Tests_Uncollected (ubuntu-latest) +``` + +**Class Mode Example**: +``` +split_tests_lin / Aspire.Templates.Tests_BuildAndRunTemplateTests (ubuntu-latest) +split_tests_lin / Aspire.Templates.Tests_EmptyTemplateRunTests (ubuntu-latest) +split_tests_lin / Aspire.Templates.Tests_StarterTemplateRunTests (ubuntu-latest) +... +``` + +Each job: +1. Downloads built packages (if `requiresNugets: true`) +2. Installs test SDK (if `requiresTestSdk: true`) +3. Runs: `dotnet test ... -- ` +4. Uploads test results + +**Expected Duration**: Varies by test, but should be significantly less than running all tests together + +## Monitoring Your PR + +### 1. Check Setup Jobs + +Navigate to your PR → Actions → Click on workflow run → Expand setup jobs + +**What to Look For**: +- [ ] "Build split test projects" step succeeds +- [ ] "Load split tests matrix" step outputs JSON +- [ ] Check artifacts → `logs-enumerate-tests-{OS}` contains: + - [ ] `.tests.list` files + - [ ] `.tests.metadata.json` files + - [ ] `split-tests-matrix.json` + +**Download and Inspect**: +```bash +# Download artifacts from GitHub UI +unzip logs-enumerate-tests-Linux.zip + +# Check generated files +cat artifacts/helix/*.tests.list +cat artifacts/helix/*.tests.metadata.json +cat artifacts/test-matrices/split-tests-matrix.json | jq . +``` + +### 2. Check Split Test Jobs + +Look for new jobs in the workflow run: + +**Collection Mode**: +- Job names like: `Split Tests Linux / {ProjectName}_Collection_{CollectionName}` +- Fewer jobs than test classes (grouped) + +**Class Mode**: +- Job names like: `Split Tests Linux / {ProjectName}_{ClassName}` +- One job per test class + +**What to Verify**: +- [ ] Jobs appear for each matrix entry +- [ ] Jobs run in parallel +- [ ] Each job uses correct filter argument +- [ ] Test results are uploaded +- [ ] All tests pass (or expected failures only) + +### 3. Compare CI Times + +**Before**: +``` +Aspire.Hosting.Tests (Linux): 1 job, 60 minutes +``` + +**After** (with collections): +``` +Collection_DatabaseTests: 25 minutes +Collection_ContainerTests: 20 minutes +Uncollected: 10 minutes +Total: ~25 minutes (parallel) +``` + +## Verification Checklist + +### Per-OS Setup (Run 3 times: Linux, macOS, Windows) + +- [ ] `setup_for_tests_{os}` job succeeds +- [ ] Split test projects are built +- [ ] Matrix JSON is generated and output +- [ ] Artifacts are uploaded + +### Split Test Execution (Per OS) + +- [ ] `split_tests_{os}` jobs appear +- [ ] Number of jobs matches matrix entries +- [ ] Each job runs correct filter +- [ ] Tests execute and pass +- [ ] Test results (.trx files) are uploaded + +### Matrix Validation + +- [ ] Download `split-tests-matrix.json` from artifacts +- [ ] Validate JSON structure: + ```bash + jq '.include | length' split-tests-matrix.json # Should be > 0 + jq '.include[0] | keys' split-tests-matrix.json # Check fields present + ``` +- [ ] Verify filter arguments are correct: + ```bash + jq '.include[] | {shortname, filterArg}' split-tests-matrix.json + ``` + +## Common CI Issues + +### Issue 1: "No split test projects found" + +**Symptom**: Setup job completes but no split_tests_* jobs run + +**Cause**: No projects have `SplitTestsOnCI=true` set + +**Fix**: Verify `.csproj` has the property set + +### Issue 2: "Matrix is empty" + +**Symptom**: split_tests_* jobs are skipped + +**Cause**: Matrix generation failed or produced empty result + +**Fix**: +1. Download artifacts +2. Check if `.tests.list` files exist +3. Check if `split-tests-matrix.json` exists and has entries +4. Review binlogs for errors + +### Issue 3: "No tests executed" + +**Symptom**: Test job completes but .trx shows 0 tests + +**Cause**: Filter argument didn't match any tests + +**Fix**: +1. Check `filterArg` in matrix JSON +2. Verify collection/class names match actual test code +3. Check `TestClassNamesPrefix` matches namespace + +### Issue 4: "Build failed for split project" + +**Symptom**: Setup job fails during "Build split test projects" + +**Cause**: Test project has build errors or missing dependencies + +**Fix**: +1. Check binlog: `Build_{ProjectName}.binlog` +2. Fix build errors +3. Test locally first with `dotnet build` + +## Rolling Back + +If issues arise in CI, you can disable splitting temporarily: + +### Option 1: Disable for One Project + +```xml + + +``` + +Push change → Project runs as single job again + +### Option 2: Disable Globally + +In `.github/workflows/tests.yml`, comment out split_tests_* jobs: + +```yaml + # split_tests_lin: + # uses: ./.github/workflows/run-tests.yml + # ... +``` + +This stops all split test execution (back to pre-PR behavior) + +## Success Metrics + +After your PR merges, track these metrics: + +### CI Time Reduction + +**Before**: Note longest test job duration +**After**: Note longest split test job duration +**Target**: 50%+ reduction + +Example: +``` +Before: Hosting.Tests = 60m +After: Collection_DatabaseTests = 25m (longest) +Improvement: 58% faster +``` + +### Job Count + +**Collection Mode**: Expect N+1 jobs (N collections + uncollected) +**Class Mode**: Expect N jobs (one per class) + +### Flakiness + +Monitor for: +- Tests failing intermittently in split jobs +- Tests passing in split jobs but failing when run together +- Resource contention issues (less likely with fewer tests per job) + +## Next Steps After CI Success + +1. **Monitor for 1-2 weeks** + - Watch for any new failures + - Check if CI times remain improved + - Look for resource issues + +2. **Enable for More Projects** + - Apply to other long-running test projects + - Add collections to optimize further + +3. **Document Learnings** + - Update best practices based on real usage + - Share collection grouping strategies + - Document any edge cases discovered + +4. **Optimize Further** + - Adjust collection groupings based on actual times + - Fine-tune timeouts + - Consider enabling for more projects \ No newline at end of file diff --git a/eng/scripts/extract-test-metadata.ps1 b/eng/scripts/extract-test-metadata.ps1 new file mode 100644 index 00000000000..3a65bb21769 --- /dev/null +++ b/eng/scripts/extract-test-metadata.ps1 @@ -0,0 +1,136 @@ +<# +.SYNOPSIS + Extract test metadata (collections or classes) from xUnit --list-tests output. + +.DESCRIPTION + Determines splitting mode: + - If any lines start with 'Collection:' (xUnit v3 collection banner) → collection mode + - Else → class mode + Outputs a .tests.list file with either: + collection:Name + ... + uncollected:* (always appended in collection mode) + OR + class:Full.Namespace.ClassName + ... + + Also updates the per-project metadata JSON with mode and collections. + +.PARAMETER TestAssemblyOutputFile + Path to a temporary file containing the raw --list-tests output (one line per entry). + +.PARAMETER TestClassNamesPrefix + Namespace prefix used to recognize test classes (e.g. Aspire.Templates.Tests). + +.PARAMETER TestCollectionsToSkip + Semicolon-separated collection names to exclude from dedicated jobs. + +.PARAMETER OutputListFile + Path to the .tests.list output file. + +.PARAMETER MetadataJsonFile + Path to the .tests.metadata.json file (script may append mode info). + +.NOTES + PowerShell 7+ + Fails fast if zero test classes discovered when in class mode. +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory=$true)] + [string]$TestAssemblyOutputFile, + + [Parameter(Mandatory=$true)] + [string]$TestClassNamesPrefix, + + [Parameter(Mandatory=$false)] + [string]$TestCollectionsToSkip = "", + + [Parameter(Mandatory=$true)] + [string]$OutputListFile, + + [Parameter(Mandatory=$false)] + [string]$MetadataJsonFile = "" +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +if (-not (Test-Path $TestAssemblyOutputFile)) { + Write-Error "TestAssemblyOutputFile not found: $TestAssemblyOutputFile" +} + +$raw = Get-Content -LiteralPath $TestAssemblyOutputFile -ErrorAction Stop + +$collections = [System.Collections.Generic.HashSet[string]]::new() +$classes = [System.Collections.Generic.HashSet[string]]::new() + +$collectionBannerRegex = '^\s*Collection:\s*(.+)$' +$classRegex = "^\s*$([Regex]::Escape($TestClassNamesPrefix))\.[^\(]+$" + +foreach ($line in $raw) { + if ($line -match $collectionBannerRegex) { + $c = $Matches[1].Trim() + if ($c) { $collections.Add($c) | Out-Null } + continue + } + if ($line -match $classRegex) { + # The line is like Namespace.ClassName.MethodName + # Reduce to Namespace.ClassName + if ($line -match '^(' + [Regex]::Escape($TestClassNamesPrefix) + '\.[^\.]+)\.') { + $classes.Add($Matches[1]) | Out-Null + } + } +} + +$skipList = @() +if ($TestCollectionsToSkip) { + $skipList = $TestCollectionsToSkip -split ';' | ForEach-Object { $_.Trim() } | Where-Object { $_ } +} + +$filteredCollections = $collections | Where-Object { $skipList -notcontains $_ } + +$mode = if ($filteredCollections.Count -gt 0) { 'collection' } else { 'class' } + +if ($classes.Count -eq 0 -and $mode -eq 'class') { + Write-Error "No test classes discovered matching prefix '$TestClassNamesPrefix'." +} + +$outputDir = [System.IO.Path]::GetDirectoryName($OutputListFile) +if ($outputDir -and -not (Test-Path $outputDir)) { + New-Item -ItemType Directory -Path $outputDir -Force | Out-Null +} + +$lines = [System.Collections.Generic.List[string]]::new() + +if ($mode -eq 'collection') { + foreach ($c in ($filteredCollections | Sort-Object)) { + $lines.Add("collection:$c") + } + $lines.Add("uncollected:*") +} else { + foreach ($cls in ($classes | Sort-Object)) { + $lines.Add("class:$cls") + } +} + +$lines | Set-Content -Path $OutputListFile -Encoding UTF8 + +if ($MetadataJsonFile -and (Test-Path $MetadataJsonFile)) { + try { + $meta = Get-Content -Raw -Path $MetadataJsonFile | ConvertFrom-Json + $meta.mode = $mode + $meta.collections = ($filteredCollections | Sort-Object) + $meta.classCount = $classes.Count + $meta.collectionCount = $filteredCollections.Count + $meta | ConvertTo-Json -Depth 20 | Set-Content -Path $MetadataJsonFile -Encoding UTF8 + } catch { + Write-Warning "Failed updating metadata JSON: $_" + } +} + +Write-Host "Mode: $mode" +Write-Host "Collections discovered (after filtering): $($filteredCollections.Count)" +Write-Host "Classes discovered: $($classes.Count)" +Write-Host "Output list written: $OutputListFile" \ No newline at end of file diff --git a/eng/scripts/generate-test-matrix.ps1 b/eng/scripts/generate-test-matrix.ps1 new file mode 100644 index 00000000000..6cbef959cc1 --- /dev/null +++ b/eng/scripts/generate-test-matrix.ps1 @@ -0,0 +1,186 @@ +<# +.SYNOPSIS + Generate split-tests matrix JSON supporting collection-based and class-based modes. + +.DESCRIPTION + Reads *.tests.list files: + collection mode format: + collection:Name + ... + uncollected:* (catch-all) + class mode format: + class:Full.Namespace.ClassName + + Builds matrix entries with fields consumed by CI: + type (collection | uncollected | class) + projectName + shortname + name + fullClassName (class mode only) + testProjectPath + filterArg + requiresNugets + requiresTestSdk + enablePlaywrightInstall + testSessionTimeout + testHangTimeout + + Defaults (if metadata absent): + testSessionTimeout=20m + testHangTimeout=10m + uncollectedTestsSessionTimeout=15m + uncollectedTestsHangTimeout=10m + +.NOTES + PowerShell 7+, cross-platform. +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory=$true)] + [string]$TestListsDirectory, + [Parameter(Mandatory=$true)] + [string]$OutputDirectory, + [Parameter(Mandatory=$false)] + [ValidateSet('windows','linux','darwin','')] + [string]$BuildOs = '' +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +function Read-Metadata($file, $projectName) { + $defaults = @{ + projectName = $projectName + testClassNamesPrefix = $projectName + testProjectPath = "tests/$projectName/$projectName.csproj" + requiresNugets = 'false' + requiresTestSdk = 'false' + enablePlaywrightInstall = 'false' + testSessionTimeout = '20m' + testHangTimeout = '10m' + uncollectedTestsSessionTimeout = '15m' + uncollectedTestsHangTimeout = '10m' + } + if (-not (Test-Path $file)) { return $defaults } + try { + $json = Get-Content -Raw -Path $file | ConvertFrom-Json + foreach ($k in $json.PSObject.Properties.Name) { + $defaults[$k] = $json.$k + } + } catch { + Write-Warning "Failed parsing metadata for $projectName: $_" + } + return $defaults +} + +function New-EntryCollection($c,$meta) { + [ordered]@{ + type = 'collection' + projectName = $meta.projectName + name = $c + shortname = "Collection_$c" + testProjectPath = $meta.testProjectPath + filterArg = "--filter-collection `"$c`"" + requiresNugets = ($meta.requiresNugets -eq 'true') + requiresTestSdk = ($meta.requiresTestSdk -eq 'true') + enablePlaywrightInstall = ($meta.enablePlaywrightInstall -eq 'true') + testSessionTimeout = $meta.testSessionTimeout + testHangTimeout = $meta.testHangTimeout + } +} + +function New-EntryUncollected($collections,$meta) { + $filters = @() + foreach ($c in $collections) { + $filters += "--filter-not-collection `"$c`"" + } + [ordered]@{ + type = 'uncollected' + projectName = $meta.projectName + name = 'UncollectedTests' + shortname = 'Uncollected' + testProjectPath = $meta.testProjectPath + filterArg = ($filters -join ' ') + requiresNugets = ($meta.requiresNugets -eq 'true') + requiresTestSdk = ($meta.requiresTestSdk -eq 'true') + enablePlaywrightInstall = ($meta.enablePlaywrightInstall -eq 'true') + testSessionTimeout = ($meta.uncollectedTestsSessionTimeout ?? $meta.testSessionTimeout) + testHangTimeout = ($meta.uncollectedTestsHangTimeout ?? $meta.testHangTimeout) + } +} + +function New-EntryClass($full,$meta) { + $prefix = $meta.testClassNamesPrefix + $short = $full + if ($prefix -and $full.StartsWith("$prefix.")) { + $short = $full.Substring($prefix.Length + 1) + } + [ordered]@{ + type = 'class' + projectName = $meta.projectName + name = $short + shortname = $short + fullClassName = $full + testProjectPath = $meta.testProjectPath + filterArg = "--filter-class `"$full`"" + requiresNugets = ($meta.requiresNugets -eq 'true') + requiresTestSdk = ($meta.requiresTestSdk -eq 'true') + enablePlaywrightInstall = ($meta.enablePlaywrightInstall -eq 'true') + testSessionTimeout = $meta.testSessionTimeout + testHangTimeout = $meta.testHangTimeout + } +} + +if (-not (Test-Path $TestListsDirectory)) { + Write-Warning "Test lists directory not found: $TestListsDirectory" + exit 0 +} + +$listFiles = Get-ChildItem -Path $TestListsDirectory -Filter '*.tests.list' -ErrorAction SilentlyContinue +if ($listFiles.Count -eq 0) { + $empty = @{ include = @() } + New-Item -ItemType Directory -Force -Path $OutputDirectory | Out-Null + $empty | ConvertTo-Json -Depth 5 -Compress | Set-Content -Path (Join-Path $OutputDirectory 'split-tests-matrix.json') -Encoding UTF8 + Write-Host "Empty matrix written (no .tests.list files)." + exit 0 +} + +$entries = [System.Collections.Generic.List[object]]::new() + +foreach ($lf in $listFiles) { + $baseName = [System.IO.Path]::GetFileNameWithoutExtension($lf.Name -replace '\.tests$','') + $projectName = $baseName + $lines = Get-Content $lf.FullName | Where-Object { $_ -and -not [string]::IsNullOrWhiteSpace($_) } + $metadataPath = ($lf.FullName -replace '\.tests\.list$', '.tests.metadata.json') + $meta = Read-Metadata $metadataPath $projectName + if ($lines.Count -eq 0) { continue } + + if ($lines[0].StartsWith('collection:') -or $lines[0].StartsWith('uncollected:')) { + # collection mode + $collections = @() + $hasUncollected = $false + foreach ($l in $lines) { + if ($l -match '^collection:(.+)$') { $collections += $Matches[1].Trim() } + elseif ($l -match '^uncollected:') { $hasUncollected = $true } + } + foreach ($c in ($collections | Sort-Object)) { + $entries.Add( (New-EntryCollection $c $meta) ) | Out-Null + } + if ($hasUncollected) { + $entries.Add( (New-EntryUncollected $collections $meta) ) | Out-Null + } + } elseif ($lines[0].StartsWith('class:')) { + # class mode + foreach ($l in $lines) { + if ($l -match '^class:(.+)$') { + $entries.Add( (New-EntryClass $Matches[1].Trim() $meta) ) | Out-Null + } + } + } +} + +$matrix = @{ include = $entries } +New-Item -ItemType Directory -Force -Path $OutputDirectory | Out-Null +$matrix | ConvertTo-Json -Depth 10 -Compress | Set-Content -Path (Join-Path $OutputDirectory 'split-tests-matrix.json') -Encoding UTF8 +Write-Host "Matrix entries: $($entries.Count)" \ No newline at end of file From 3547eec8bc47a01011196ddffd998ebeb71d70d1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 16 Oct 2025 20:20:53 +0000 Subject: [PATCH 03/48] Initial plan From 685fcc71531478c0c5caf7e7fef0137cebe4f05d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 16 Oct 2025 20:36:53 +0000 Subject: [PATCH 04/48] Add test splitting infrastructure with auto-detection - Enhanced Directory.Build.targets with new ExtractTestClassNamesV3 target - Updated GetTestProjects.proj to support split test matrix generation - Fixed PowerShell scripts for proper array handling and cross-platform paths - Migrated Aspire.Templates.Tests to use new unified mechanism - Successfully generates class-based split test matrices Co-authored-by: radical <1472+radical@users.noreply.github.com> --- eng/scripts/extract-test-metadata.ps1 | 24 ++-- eng/scripts/generate-test-matrix.ps1 | 10 +- .../Aspire.Templates.Tests.csproj | 14 ++- tests/Directory.Build.targets | 114 +++++++++++++++++- tests/Shared/GetTestProjects.proj | 53 ++++++-- 5 files changed, 184 insertions(+), 31 deletions(-) diff --git a/eng/scripts/extract-test-metadata.ps1 b/eng/scripts/extract-test-metadata.ps1 index 3a65bb21769..fddff3e64e2 100644 --- a/eng/scripts/extract-test-metadata.ps1 +++ b/eng/scripts/extract-test-metadata.ps1 @@ -67,7 +67,7 @@ $collections = [System.Collections.Generic.HashSet[string]]::new() $classes = [System.Collections.Generic.HashSet[string]]::new() $collectionBannerRegex = '^\s*Collection:\s*(.+)$' -$classRegex = "^\s*$([Regex]::Escape($TestClassNamesPrefix))\.[^\(]+$" +$classNamePattern = '^(\s*)' + [Regex]::Escape($TestClassNamesPrefix) + '\.([^\.]+)\.' foreach ($line in $raw) { if ($line -match $collectionBannerRegex) { @@ -75,12 +75,11 @@ foreach ($line in $raw) { if ($c) { $collections.Add($c) | Out-Null } continue } - if ($line -match $classRegex) { - # The line is like Namespace.ClassName.MethodName - # Reduce to Namespace.ClassName - if ($line -match '^(' + [Regex]::Escape($TestClassNamesPrefix) + '\.[^\.]+)\.') { - $classes.Add($Matches[1]) | Out-Null - } + # Extract class name from test name + # Format: " Namespace.ClassName.MethodName(...)" or "Namespace.ClassName.MethodName" + if ($line -match $classNamePattern) { + $className = "$TestClassNamesPrefix.$($Matches[2])" + $classes.Add($className) | Out-Null } } @@ -89,7 +88,7 @@ if ($TestCollectionsToSkip) { $skipList = $TestCollectionsToSkip -split ';' | ForEach-Object { $_.Trim() } | Where-Object { $_ } } -$filteredCollections = $collections | Where-Object { $skipList -notcontains $_ } +$filteredCollections = @($collections | Where-Object { $skipList -notcontains $_ }) $mode = if ($filteredCollections.Count -gt 0) { 'collection' } else { 'class' } @@ -120,10 +119,11 @@ $lines | Set-Content -Path $OutputListFile -Encoding UTF8 if ($MetadataJsonFile -and (Test-Path $MetadataJsonFile)) { try { $meta = Get-Content -Raw -Path $MetadataJsonFile | ConvertFrom-Json - $meta.mode = $mode - $meta.collections = ($filteredCollections | Sort-Object) - $meta.classCount = $classes.Count - $meta.collectionCount = $filteredCollections.Count + # Add or update properties + $meta | Add-Member -Force -MemberType NoteProperty -Name 'mode' -Value $mode + $meta | Add-Member -Force -MemberType NoteProperty -Name 'collections' -Value @($filteredCollections | Sort-Object) + $meta | Add-Member -Force -MemberType NoteProperty -Name 'classCount' -Value $classes.Count + $meta | Add-Member -Force -MemberType NoteProperty -Name 'collectionCount' -Value $filteredCollections.Count $meta | ConvertTo-Json -Depth 20 | Set-Content -Path $MetadataJsonFile -Encoding UTF8 } catch { Write-Warning "Failed updating metadata JSON: $_" diff --git a/eng/scripts/generate-test-matrix.ps1 b/eng/scripts/generate-test-matrix.ps1 index 6cbef959cc1..165b0fed7b8 100644 --- a/eng/scripts/generate-test-matrix.ps1 +++ b/eng/scripts/generate-test-matrix.ps1 @@ -69,7 +69,7 @@ function Read-Metadata($file, $projectName) { $defaults[$k] = $json.$k } } catch { - Write-Warning "Failed parsing metadata for $projectName: $_" + Write-Warning "Failed parsing metadata for ${projectName}: $_" } return $defaults } @@ -137,7 +137,7 @@ if (-not (Test-Path $TestListsDirectory)) { exit 0 } -$listFiles = Get-ChildItem -Path $TestListsDirectory -Filter '*.tests.list' -ErrorAction SilentlyContinue +$listFiles = @(Get-ChildItem -Path $TestListsDirectory -Filter '*.tests.list' -Recurse -ErrorAction SilentlyContinue) if ($listFiles.Count -eq 0) { $empty = @{ include = @() } New-Item -ItemType Directory -Force -Path $OutputDirectory | Out-Null @@ -149,9 +149,9 @@ if ($listFiles.Count -eq 0) { $entries = [System.Collections.Generic.List[object]]::new() foreach ($lf in $listFiles) { - $baseName = [System.IO.Path]::GetFileNameWithoutExtension($lf.Name -replace '\.tests$','') - $projectName = $baseName - $lines = Get-Content $lf.FullName | Where-Object { $_ -and -not [string]::IsNullOrWhiteSpace($_) } + $fileName = $lf.Name -replace '\.tests\.list$','' + $projectName = $fileName + $lines = @(Get-Content $lf.FullName | Where-Object { $_ -and -not [string]::IsNullOrWhiteSpace($_) }) $metadataPath = ($lf.FullName -replace '\.tests\.list$', '.tests.metadata.json') $meta = Read-Metadata $metadataPath $projectName if ($lines.Count -eq 0) { continue } diff --git a/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj b/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj index cfd44eedfd5..16de814c464 100644 --- a/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj +++ b/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj @@ -13,8 +13,18 @@ true true - true - Aspire.Templates.Tests + + true + Aspire.Templates.Tests + + + true + true + true + + + 20m + 12m + + + + + + + + + + + + + + + + <_DiscoveryScriptPath>$(RepoRoot)eng\scripts\extract-test-metadata.ps1 + + + <_TestListFile>$(TestArchiveTestsDir)$(MSBuildProjectName).tests.list + <_MetadataFile>$(TestArchiveTestsDir)$(MSBuildProjectName).tests.metadata.json + + + <_RelativeProjectPath>$(MSBuildProjectDirectory.Replace('$(RepoRoot)', '')) + <_RelativeProjectPath>$(_RelativeProjectPath.Replace('\', '/')) + + + <_CollectionsToSkip Condition="'$(TestCollectionsToSkipSplitting)' != ''">$(TestCollectionsToSkipSplitting) + <_CollectionsToSkip Condition="'$(TestCollectionsToSkipSplitting)' == ''"> + + + + + + + + <_TempOutputFile>$(TestArchiveTestsDir)$(MSBuildProjectName).tests.output.tmp + + + + + + + <_InitialMetadataLines Include="{" /> + <_InitialMetadataLines Include=" "projectName": "$(MSBuildProjectName)"," /> + <_InitialMetadataLines Include=" "testClassNamesPrefix": "$(TestClassNamesPrefix)"," /> + <_InitialMetadataLines Include=" "testProjectPath": "$(_RelativeProjectPath)/$(MSBuildProjectFile)"," /> + <_InitialMetadataLines Include=" "requiresNugets": "$(RequiresNugetsForSplitTests)"," /> + <_InitialMetadataLines Include=" "requiresTestSdk": "$(RequiresTestSdkForSplitTests)"," /> + <_InitialMetadataLines Include=" "testSessionTimeout": "$(SplitTestSessionTimeout)"," /> + <_InitialMetadataLines Include=" "testHangTimeout": "$(SplitTestHangTimeout)"," /> + <_InitialMetadataLines Include=" "uncollectedTestsSessionTimeout": "$(UncollectedTestsSessionTimeout)"," /> + <_InitialMetadataLines Include=" "uncollectedTestsHangTimeout": "$(UncollectedTestsHangTimeout)"," /> + <_InitialMetadataLines Include=" "enablePlaywrightInstall": "$(EnablePlaywrightInstallForSplitTests)"" /> + <_InitialMetadataLines Include="}" /> + + + + + + + + + <_PwshCommand>pwsh + <_PwshCommand Condition="'$(OS)' == 'Windows_NT'">powershell + <_DiscoveryCommand>$(_PwshCommand) -NoProfile -ExecutionPolicy Bypass -File "$(_DiscoveryScriptPath)" + <_DiscoveryCommand>$(_DiscoveryCommand) -TestAssemblyOutputFile "$(_TempOutputFile)" + <_DiscoveryCommand>$(_DiscoveryCommand) -TestClassNamesPrefix "$(TestClassNamesPrefix)" + <_DiscoveryCommand Condition="'$(_CollectionsToSkip)' != ''">$(_DiscoveryCommand) -TestCollectionsToSkip "$(_CollectionsToSkip)" + <_DiscoveryCommand>$(_DiscoveryCommand) -OutputListFile "$(_TestListFile)" + <_DiscoveryCommand>$(_DiscoveryCommand) -MetadataJsonFile "$(_MetadataFile)" + + + + + + + + + + + + + + - - - + + + diff --git a/tests/Shared/GetTestProjects.proj b/tests/Shared/GetTestProjects.proj index 1b280fdf065..25f8159c9c9 100644 --- a/tests/Shared/GetTestProjects.proj +++ b/tests/Shared/GetTestProjects.proj @@ -4,16 +4,18 @@ It emits "shortnames" for the project which is essentially the project name with `Aspire.` prefix `.Tests` removed. - Input: $(TestsListOutputPath) --> + Input: $(TestsListOutputPath) + Optional: $(TestMatrixOutputPath) - if set, generates split test matrices --> - + + $(MSBuildThisFileDirectory)..\..\ + + + - - $(MSBuildThisFileDirectory)..\..\ - - <_TestProjectsToExclude Include="$(RepoRoot)tests\Shared\**\*Tests.csproj" /> <_TestProjectsToExclude Include="$(RepoRoot)tests\testproject\**\*Tests.csproj" /> @@ -36,13 +38,50 @@ + + + + + + + + + + + + <_TestListsDir>$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix')) + <_BuildOs Condition="'$(BuildOs)' == ''">unknown + <_BuildOs Condition="'$(BuildOs)' != ''">$(BuildOs) + + + + + + + <_PwshCommand>pwsh + <_PwshCommand Condition="'$(OS)' == 'Windows_NT'">powershell + <_MatrixScriptPath>$([MSBuild]::NormalizePath($(RepoRoot), 'eng', 'scripts', 'generate-test-matrix.ps1')) + <_MatrixCommand>$(_PwshCommand) -NoProfile -ExecutionPolicy Bypass -File "$(_MatrixScriptPath)" + <_MatrixCommand>$(_MatrixCommand) -TestListsDirectory "$(_TestListsDir)" + <_MatrixCommand>$(_MatrixCommand) -OutputDirectory "$(TestMatrixOutputPath)" + <_MatrixCommand>$(_MatrixCommand) -BuildOs "$(_BuildOs)" + + + From 8902ea5853c473c2e97078d361b93c14f8f40ef7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 16 Oct 2025 20:42:17 +0000 Subject: [PATCH 05/48] Fix path normalization in GetTestProjects.proj - Use MSBuild::NormalizeDirectory for cross-platform compatibility - Define ArtifactsDir if not already set - Matrix generation now works correctly with normalized paths Co-authored-by: radical <1472+radical@users.noreply.github.com> --- tests/Shared/GetTestProjects.proj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/Shared/GetTestProjects.proj b/tests/Shared/GetTestProjects.proj index 25f8159c9c9..6fd3a457e40 100644 --- a/tests/Shared/GetTestProjects.proj +++ b/tests/Shared/GetTestProjects.proj @@ -8,7 +8,8 @@ Optional: $(TestMatrixOutputPath) - if set, generates split test matrices --> - $(MSBuildThisFileDirectory)..\..\ + $([MSBuild]::NormalizeDirectory($(MSBuildThisFileDirectory), '..', '..')) + $([MSBuild]::NormalizeDirectory($(RepoRoot), 'artifacts')) Date: Thu, 16 Oct 2025 21:17:32 +0000 Subject: [PATCH 06/48] Enable test splitting for Aspire.Hosting.Tests and integrate with GitHub Actions - Added [Collection] attributes to WaitForTests and DistributedApplicationTests classes - Enabled SplitTestsOnCI for Aspire.Hosting.Tests project with appropriate timeouts - Updated enumerate-tests action to generate and output split tests matrix - Added split test jobs to tests.yml workflow for Linux, macOS, and Windows - Matrix generates 66 class-based jobs for Hosting.Tests (40 Hosting + 26 Templates) - Filter arguments validated: --filter-class correctly isolates test classes Co-authored-by: radical <1472+radical@users.noreply.github.com> --- .github/actions/enumerate-tests/action.yml | 24 +++++++ .github/workflows/tests.yml | 68 ++++++++++++++++++- .../Aspire.Hosting.Tests.csproj | 10 +++ .../DistributedApplicationTests.cs | 1 + tests/Aspire.Hosting.Tests/WaitForTests.cs | 1 + 5 files changed, 103 insertions(+), 1 deletion(-) diff --git a/.github/actions/enumerate-tests/action.yml b/.github/actions/enumerate-tests/action.yml index 3db315b0f17..a8b2fe6c549 100644 --- a/.github/actions/enumerate-tests/action.yml +++ b/.github/actions/enumerate-tests/action.yml @@ -17,6 +17,9 @@ outputs: templates_tests_matrix: description: Templates tests matrix value: ${{ steps.generate_templates_matrix.outputs.templates_tests_matrix }} + split_tests_matrix: + description: Split tests matrix (for collection/class-based splitting) + value: ${{ steps.generate_split_tests_matrix.outputs.split_tests_matrix }} runs: using: "composite" steps: @@ -35,7 +38,9 @@ runs: dotnet build ${{ github.workspace }}/tests/Shared/GetTestProjects.proj /bl:${{ github.workspace }}/artifacts/log/Debug/GetTestProjects.binlog /p:TestsListOutputPath=${{ github.workspace }}/artifacts/TestsForGithubActions.list + /p:TestMatrixOutputPath=${{ github.workspace }}/artifacts/test-matrices/ /p:ContinuousIntegrationBuild=true + /p:BuildOs=${{ runner.os == 'Linux' && 'linux' || runner.os == 'macOS' && 'darwin' || 'windows' }} - name: Generate list of template tests if: ${{ inputs.includeTemplates }} @@ -83,6 +88,25 @@ runs: "templates_tests_matrix=$jsonString" "templates_tests_matrix=$jsonString" | Out-File -FilePath $env:GITHUB_OUTPUT + - name: Generate split tests matrix + id: generate_split_tests_matrix + if: ${{ inputs.includeIntegrations }} + shell: pwsh + run: | + $matrixFilePath = "${{ github.workspace }}/artifacts/test-matrices/split-tests-matrix.json" + if (Test-Path $matrixFilePath) { + $matrixContent = Get-Content -Raw $matrixFilePath + # Validate it's valid JSON + $null = $matrixContent | ConvertFrom-Json + "split_tests_matrix=$matrixContent" | Out-File -FilePath $env:GITHUB_OUTPUT + Write-Host "Split tests matrix generated successfully" + } else { + # Empty matrix if no split tests + $emptyMatrix = '{"include":[]}' + "split_tests_matrix=$emptyMatrix" | Out-File -FilePath $env:GITHUB_OUTPUT + Write-Host "No split tests matrix found, using empty matrix" + } + - name: Upload logs if: always() uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a4cd05fa553..5e4e91b6925 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,6 +18,7 @@ jobs: outputs: integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }} templates_tests_matrix: ${{ steps.generate_tests_matrix.outputs.templates_tests_matrix }} + split_tests_matrix: ${{ steps.generate_tests_matrix.outputs.split_tests_matrix }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -33,6 +34,7 @@ jobs: outputs: integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }} templates_tests_matrix: ${{ steps.generate_tests_matrix.outputs.templates_tests_matrix }} + split_tests_matrix: ${{ steps.generate_tests_matrix.outputs.split_tests_matrix }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -48,6 +50,7 @@ jobs: outputs: integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }} templates_tests_matrix: ${{ steps.generate_tests_matrix.outputs.templates_tests_matrix }} + split_tests_matrix: ${{ steps.generate_tests_matrix.outputs.split_tests_matrix }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -161,6 +164,66 @@ jobs: requiresNugets: true requiresTestSdk: true + split_tests_lin: + name: Split Linux + uses: ./.github/workflows/run-tests.yml + needs: [setup_for_tests_lin, build_packages] + if: ${{ fromJson(needs.setup_for_tests_lin.outputs.split_tests_matrix).include[0] != null }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup_for_tests_lin.outputs.split_tests_matrix) }} + with: + testShortName: ${{ matrix.projectName }}_${{ matrix.shortname }} + os: "ubuntu-latest" + testProjectPath: ${{ matrix.testProjectPath }} + testSessionTimeout: ${{ matrix.testSessionTimeout }} + testHangTimeout: ${{ matrix.testHangTimeout }} + extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true ${{ matrix.filterArg }}" + versionOverrideArg: ${{ inputs.versionOverrideArg }} + requiresNugets: ${{ matrix.requiresNugets }} + requiresTestSdk: ${{ matrix.requiresTestSdk }} + enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} + + split_tests_macos: + name: Split macos + uses: ./.github/workflows/run-tests.yml + needs: [setup_for_tests_macos, build_packages] + if: ${{ fromJson(needs.setup_for_tests_macos.outputs.split_tests_matrix).include[0] != null }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup_for_tests_macos.outputs.split_tests_matrix) }} + with: + testShortName: ${{ matrix.projectName }}_${{ matrix.shortname }} + os: "macos-latest" + testProjectPath: ${{ matrix.testProjectPath }} + testSessionTimeout: ${{ matrix.testSessionTimeout }} + testHangTimeout: ${{ matrix.testHangTimeout }} + extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true ${{ matrix.filterArg }}" + versionOverrideArg: ${{ inputs.versionOverrideArg }} + requiresNugets: ${{ matrix.requiresNugets }} + requiresTestSdk: ${{ matrix.requiresTestSdk }} + enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} + + split_tests_win: + name: Split Windows + uses: ./.github/workflows/run-tests.yml + needs: [setup_for_tests_win, build_packages] + if: ${{ fromJson(needs.setup_for_tests_win.outputs.split_tests_matrix).include[0] != null }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup_for_tests_win.outputs.split_tests_matrix) }} + with: + testShortName: ${{ matrix.projectName }}_${{ matrix.shortname }} + os: "windows-latest" + testProjectPath: ${{ matrix.testProjectPath }} + testSessionTimeout: ${{ matrix.testSessionTimeout }} + testHangTimeout: ${{ matrix.testHangTimeout }} + extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true ${{ matrix.filterArg }}" + versionOverrideArg: ${{ inputs.versionOverrideArg }} + requiresNugets: ${{ matrix.requiresNugets }} + requiresTestSdk: ${{ matrix.requiresTestSdk }} + enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} + endtoend_tests: name: EndToEnd Linux uses: ./.github/workflows/run-tests.yml @@ -210,7 +273,10 @@ jobs: integrations_test_win, templates_test_lin, templates_test_macos, - templates_test_win + templates_test_win, + split_tests_lin, + split_tests_macos, + split_tests_win ] steps: - name: Checkout code diff --git a/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj b/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj index f55080a25d1..b3a9e27f224 100644 --- a/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj +++ b/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj @@ -10,6 +10,16 @@ false false + + + true + Aspire.Hosting.Tests + + + 30m + 15m + 20m + 10m diff --git a/tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs b/tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs index 8470c439756..ce49ca17063 100644 --- a/tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs +++ b/tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs @@ -29,6 +29,7 @@ namespace Aspire.Hosting.Tests; +[Collection("DistributedApplicationTests")] public class DistributedApplicationTests { private readonly ITestOutputHelper _testOutputHelper; diff --git a/tests/Aspire.Hosting.Tests/WaitForTests.cs b/tests/Aspire.Hosting.Tests/WaitForTests.cs index 559d8f703d8..6babef39d22 100644 --- a/tests/Aspire.Hosting.Tests/WaitForTests.cs +++ b/tests/Aspire.Hosting.Tests/WaitForTests.cs @@ -11,6 +11,7 @@ namespace Aspire.Hosting.Tests; +[Collection("WaitForTests")] public class WaitForTests(ITestOutputHelper testOutputHelper) { [Fact] From b353caf7f666fd5f93736ecce1cd67088e00b681 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 16 Oct 2025 22:23:35 +0000 Subject: [PATCH 07/48] Fix test splitting issues and add ExtractTestPartitions tool Issue 1: Fixed duplicate "class:" prefix in filterArg - Updated enumerate-tests action to strip "class:", "collection:", or "uncollected:" prefixes from shortnames Issue 2: Implemented ExtractTestPartitions tool for collection/partition detection - Created tool in tools/ExtractTestPartitions to extract [Collection] and [Trait("Partition", ...)] attributes - Updated extract-test-metadata.ps1 to call the tool before falling back to class-based mode - Updated Directory.Build.targets to pass TestAssemblyPath and RepoRoot parameters Added [Trait("Partition", ...)] attributes to test classes: - WaitForTests - DistributedApplicationTests - SlimTestProgramTests Updated filter generation: - Collection filters now use "--filter-trait \"Partition=Name\"" format - Uncollected filters use "--filter-not-trait \"Partition=Name\"" format Validation: - Matrix generates 30 jobs (4 collections for Hosting + 26 classes for Templates) - Filters tested and work correctly with xUnit v3 Co-authored-by: radical <1472+radical@users.noreply.github.com> --- .github/actions/enumerate-tests/action.yml | 5 +- eng/scripts/extract-test-metadata.ps1 | 70 ++++++++-- eng/scripts/generate-test-matrix.ps1 | 4 +- .../DistributedApplicationTests.cs | 1 + .../SlimTestProgramTests.cs | 1 + tests/Aspire.Hosting.Tests/WaitForTests.cs | 1 + tests/Directory.Build.targets | 3 + .../ExtractTestPartitions.csproj | 10 ++ tools/ExtractTestPartitions/Program.cs | 126 ++++++++++++++++++ 9 files changed, 207 insertions(+), 14 deletions(-) create mode 100644 tools/ExtractTestPartitions/ExtractTestPartitions.csproj create mode 100644 tools/ExtractTestPartitions/Program.cs diff --git a/.github/actions/enumerate-tests/action.yml b/.github/actions/enumerate-tests/action.yml index a8b2fe6c549..c132f9bd518 100644 --- a/.github/actions/enumerate-tests/action.yml +++ b/.github/actions/enumerate-tests/action.yml @@ -78,7 +78,10 @@ runs: $prefix = "Aspire.Templates.Tests." $lines = Get-Content $inputFilePath | ForEach-Object { - $_ -replace "^$prefix", "" + # Strip "class:" or "collection:" prefix first + $line = $_ -replace "^(class|collection|uncollected):", "" + # Then strip the test class prefix + $line -replace "^$prefix", "" } $jsonObject = @{ diff --git a/eng/scripts/extract-test-metadata.ps1 b/eng/scripts/extract-test-metadata.ps1 index fddff3e64e2..ae507f1bc14 100644 --- a/eng/scripts/extract-test-metadata.ps1 +++ b/eng/scripts/extract-test-metadata.ps1 @@ -1,10 +1,11 @@ <# .SYNOPSIS - Extract test metadata (collections or classes) from xUnit --list-tests output. + Extract test metadata (collections or classes) from test assemblies. .DESCRIPTION - Determines splitting mode: - - If any lines start with 'Collection:' (xUnit v3 collection banner) → collection mode + Determines splitting mode by extracting Collection and Trait attributes from the test assembly: + - Uses ExtractTestPartitions tool to find [Collection("name")] or [Trait("Partition", "name")] attributes + - If partitions found → partition mode (collections) - Else → class mode Outputs a .tests.list file with either: collection:Name @@ -19,6 +20,9 @@ .PARAMETER TestAssemblyOutputFile Path to a temporary file containing the raw --list-tests output (one line per entry). +.PARAMETER TestAssemblyPath + Path to the test assembly DLL for extracting partition attributes. + .PARAMETER TestClassNamesPrefix Namespace prefix used to recognize test classes (e.g. Aspire.Templates.Tests). @@ -31,6 +35,9 @@ .PARAMETER MetadataJsonFile Path to the .tests.metadata.json file (script may append mode info). +.PARAMETER RepoRoot + Path to the repository root (for locating the ExtractTestPartitions tool). + .NOTES PowerShell 7+ Fails fast if zero test classes discovered when in class mode. @@ -41,6 +48,9 @@ param( [Parameter(Mandatory=$true)] [string]$TestAssemblyOutputFile, + [Parameter(Mandatory=$true)] + [string]$TestAssemblyPath, + [Parameter(Mandatory=$true)] [string]$TestClassNamesPrefix, @@ -51,7 +61,10 @@ param( [string]$OutputListFile, [Parameter(Mandatory=$false)] - [string]$MetadataJsonFile = "" + [string]$MetadataJsonFile = "", + + [Parameter(Mandatory=$true)] + [string]$RepoRoot ) $ErrorActionPreference = 'Stop' @@ -66,15 +79,50 @@ $raw = Get-Content -LiteralPath $TestAssemblyOutputFile -ErrorAction Stop $collections = [System.Collections.Generic.HashSet[string]]::new() $classes = [System.Collections.Generic.HashSet[string]]::new() -$collectionBannerRegex = '^\s*Collection:\s*(.+)$' -$classNamePattern = '^(\s*)' + [Regex]::Escape($TestClassNamesPrefix) + '\.([^\.]+)\.' +# Extract partitions using the ExtractTestPartitions tool +$partitionsFile = [System.IO.Path]::GetTempFileName() +try { + $toolPath = Join-Path $RepoRoot "artifacts/bin/ExtractTestPartitions/Debug/net8.0/ExtractTestPartitions.dll" + + # Build the tool if it doesn't exist + if (-not (Test-Path $toolPath)) { + Write-Host "Building ExtractTestPartitions tool..." + $toolProjectPath = Join-Path $RepoRoot "tools/ExtractTestPartitions/ExtractTestPartitions.csproj" + & dotnet build $toolProjectPath -c Debug --nologo -v quiet + if ($LASTEXITCODE -ne 0) { + Write-Warning "Failed to build ExtractTestPartitions tool. Falling back to class-based mode." + } + } -foreach ($line in $raw) { - if ($line -match $collectionBannerRegex) { - $c = $Matches[1].Trim() - if ($c) { $collections.Add($c) | Out-Null } - continue + # Run the tool if available + if (Test-Path $toolPath) { + Write-Host "Extracting partitions from assembly: $TestAssemblyPath" + & dotnet $toolPath --assembly-path $TestAssemblyPath --output-file $partitionsFile 2>&1 | Write-Host + + if ($LASTEXITCODE -eq 0 -and (Test-Path $partitionsFile)) { + $partitionLines = Get-Content $partitionsFile -ErrorAction SilentlyContinue + if ($partitionLines) { + foreach ($partition in $partitionLines) { + if (-not [string]::IsNullOrWhiteSpace($partition)) { + $collections.Add($partition.Trim()) | Out-Null + } + } + Write-Host "Found $($collections.Count) partition(s) via attribute extraction" + } + } + } +} catch { + Write-Warning "Error running ExtractTestPartitions tool: $_" +} finally { + if (Test-Path $partitionsFile) { + Remove-Item $partitionsFile -Force } +} + +# Extract class names from test listing +$classNamePattern = '^(\s*)' + [Regex]::Escape($TestClassNamesPrefix) + '\.([^\.]+)\.' + +foreach ($line in $raw) { # Extract class name from test name # Format: " Namespace.ClassName.MethodName(...)" or "Namespace.ClassName.MethodName" if ($line -match $classNamePattern) { diff --git a/eng/scripts/generate-test-matrix.ps1 b/eng/scripts/generate-test-matrix.ps1 index 165b0fed7b8..06781a271e5 100644 --- a/eng/scripts/generate-test-matrix.ps1 +++ b/eng/scripts/generate-test-matrix.ps1 @@ -81,7 +81,7 @@ function New-EntryCollection($c,$meta) { name = $c shortname = "Collection_$c" testProjectPath = $meta.testProjectPath - filterArg = "--filter-collection `"$c`"" + filterArg = "--filter-trait `"Partition=$c`"" requiresNugets = ($meta.requiresNugets -eq 'true') requiresTestSdk = ($meta.requiresTestSdk -eq 'true') enablePlaywrightInstall = ($meta.enablePlaywrightInstall -eq 'true') @@ -93,7 +93,7 @@ function New-EntryCollection($c,$meta) { function New-EntryUncollected($collections,$meta) { $filters = @() foreach ($c in $collections) { - $filters += "--filter-not-collection `"$c`"" + $filters += "--filter-not-trait `"Partition=$c`"" } [ordered]@{ type = 'uncollected' diff --git a/tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs b/tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs index ce49ca17063..065090a5950 100644 --- a/tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs +++ b/tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs @@ -30,6 +30,7 @@ namespace Aspire.Hosting.Tests; [Collection("DistributedApplicationTests")] +[Trait("Partition", "DistributedApplicationTests")] public class DistributedApplicationTests { private readonly ITestOutputHelper _testOutputHelper; diff --git a/tests/Aspire.Hosting.Tests/SlimTestProgramTests.cs b/tests/Aspire.Hosting.Tests/SlimTestProgramTests.cs index ab3dd81ccc1..2348849c256 100644 --- a/tests/Aspire.Hosting.Tests/SlimTestProgramTests.cs +++ b/tests/Aspire.Hosting.Tests/SlimTestProgramTests.cs @@ -8,6 +8,7 @@ namespace Aspire.Hosting.Tests; [Collection("SlimTestProgram")] +[Trait("Partition", "SlimTestProgram")] public class SlimTestProgramTests { private readonly SlimTestProgramFixture _slimTestProgramFixture; diff --git a/tests/Aspire.Hosting.Tests/WaitForTests.cs b/tests/Aspire.Hosting.Tests/WaitForTests.cs index 6babef39d22..e6639bbafab 100644 --- a/tests/Aspire.Hosting.Tests/WaitForTests.cs +++ b/tests/Aspire.Hosting.Tests/WaitForTests.cs @@ -12,6 +12,7 @@ namespace Aspire.Hosting.Tests; [Collection("WaitForTests")] +[Trait("Partition", "WaitForTests")] public class WaitForTests(ITestOutputHelper testOutputHelper) { [Fact] diff --git a/tests/Directory.Build.targets b/tests/Directory.Build.targets index b6c0420c6b9..01d6925f518 100644 --- a/tests/Directory.Build.targets +++ b/tests/Directory.Build.targets @@ -140,12 +140,15 @@ <_PwshCommand>pwsh <_PwshCommand Condition="'$(OS)' == 'Windows_NT'">powershell + <_TestAssemblyPath>$(TargetDir)$(TargetFileName) <_DiscoveryCommand>$(_PwshCommand) -NoProfile -ExecutionPolicy Bypass -File "$(_DiscoveryScriptPath)" <_DiscoveryCommand>$(_DiscoveryCommand) -TestAssemblyOutputFile "$(_TempOutputFile)" + <_DiscoveryCommand>$(_DiscoveryCommand) -TestAssemblyPath "$(_TestAssemblyPath)" <_DiscoveryCommand>$(_DiscoveryCommand) -TestClassNamesPrefix "$(TestClassNamesPrefix)" <_DiscoveryCommand Condition="'$(_CollectionsToSkip)' != ''">$(_DiscoveryCommand) -TestCollectionsToSkip "$(_CollectionsToSkip)" <_DiscoveryCommand>$(_DiscoveryCommand) -OutputListFile "$(_TestListFile)" <_DiscoveryCommand>$(_DiscoveryCommand) -MetadataJsonFile "$(_MetadataFile)" + <_DiscoveryCommand>$(_DiscoveryCommand) -RepoRoot "$(RepoRoot)" + + + Exe + $(DefaultTargetFramework) + enable + enable + + + diff --git a/tools/ExtractTestPartitions/Program.cs b/tools/ExtractTestPartitions/Program.cs new file mode 100644 index 00000000000..b559007c09a --- /dev/null +++ b/tools/ExtractTestPartitions/Program.cs @@ -0,0 +1,126 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Reflection; + +if (args.Length < 4 || args[0] != "--assembly-path" || args[2] != "--output-file") +{ + Console.Error.WriteLine("Usage: ExtractTestPartitions --assembly-path --output-file "); + return 1; +} + +var assemblyPath = args[1]; +var outputFile = args[3]; + +ExtractPartitions(assemblyPath, outputFile); +return 0; + +static void ExtractPartitions(string assemblyPath, string outputFile) +{ + if (!File.Exists(assemblyPath)) + { + Console.Error.WriteLine($"Error: Assembly file not found: {assemblyPath}"); + Environment.Exit(1); + } + + var partitions = new HashSet(StringComparer.OrdinalIgnoreCase); + + try + { + // Load the assembly using Assembly.LoadFrom + // We need to set up an assembly resolve handler for dependencies + var assemblyDirectory = Path.GetDirectoryName(assemblyPath)!; + AppDomain.CurrentDomain.AssemblyResolve += (sender, args) => + { + var assemblyName = new AssemblyName(args.Name); + var dllPath = Path.Combine(assemblyDirectory, assemblyName.Name + ".dll"); + if (File.Exists(dllPath)) + { + return Assembly.LoadFrom(dllPath); + } + return null; + }; + + var assembly = Assembly.LoadFrom(assemblyPath); + Console.WriteLine($"Loaded assembly: {assembly.FullName}"); + + // Iterate through all types in the assembly + Type[] types; + try + { + types = assembly.GetTypes(); + } + catch (ReflectionTypeLoadException ex) + { + // Some types couldn't be loaded due to missing dependencies + // Use the types that did load + types = ex.Types.Where(t => t != null).ToArray()!; + Console.WriteLine($"Warning: Some types could not be loaded. Loaded {types.Length} types successfully."); + } + + foreach (var type in types) + { + // Check if type has Collection or Trait attributes + var attributes = type.GetCustomAttributesData(); + + foreach (var attr in attributes) + { + var attrTypeName = attr.AttributeType.FullName ?? attr.AttributeType.Name; + + // Check for [Collection("name")] attribute + if (attrTypeName.EndsWith(".CollectionAttribute") || attrTypeName == "CollectionAttribute") + { + if (attr.ConstructorArguments.Count > 0 && attr.ConstructorArguments[0].Value is string collectionName) + { + if (!string.IsNullOrWhiteSpace(collectionName)) + { + partitions.Add(collectionName); + Console.WriteLine($"Found Collection: {collectionName} on {type.Name}"); + } + } + } + // Check for [Trait("Partition", "name")] attribute + else if (attrTypeName.EndsWith(".TraitAttribute") || attrTypeName == "TraitAttribute") + { + if (attr.ConstructorArguments.Count >= 2) + { + var key = attr.ConstructorArguments[0].Value as string; + var value = attr.ConstructorArguments[1].Value as string; + + if (key?.Equals("Partition", StringComparison.OrdinalIgnoreCase) == true && + !string.IsNullOrWhiteSpace(value)) + { + partitions.Add(value); + Console.WriteLine($"Found Trait Partition: {value} on {type.Name}"); + } + } + } + } + } + + Console.WriteLine($"Total unique partitions found: {partitions.Count}"); + + // Write partitions to output file + var outputDir = Path.GetDirectoryName(outputFile); + if (!string.IsNullOrEmpty(outputDir) && !Directory.Exists(outputDir)) + { + Directory.CreateDirectory(outputDir); + } + + if (partitions.Count > 0) + { + File.WriteAllLines(outputFile, partitions.OrderBy(p => p, StringComparer.OrdinalIgnoreCase)); + Console.WriteLine($"Partitions written to: {outputFile}"); + } + else + { + Console.WriteLine("No partitions found. Not creating output file."); + } + } + catch (Exception ex) + { + Console.Error.WriteLine($"Error extracting partitions: {ex.Message}"); + Console.Error.WriteLine($"Stack trace: {ex.StackTrace}"); + Environment.Exit(1); + } +} From 862e2c9652363997fdcca035598659e0a5f7145c Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Thu, 16 Oct 2025 21:00:08 -0400 Subject: [PATCH 08/48] wip --- eng/Testing.targets | 19 ++++++++++ tests/Shared/GetTestProjects.proj | 59 ++++++++++++++++++++++++++----- 2 files changed, 69 insertions(+), 9 deletions(-) diff --git a/eng/Testing.targets b/eng/Testing.targets index 853a3949d3b..4530ea59993 100644 --- a/eng/Testing.targets +++ b/eng/Testing.targets @@ -109,4 +109,23 @@ + + + + <_ShouldSplit>false + <_ShouldSplit Condition="'$(SplitTestsOnCI)' == 'true'">true + + + + <_ProjectInfo Include="$(MSBuildProjectFullPath)"> + $(RunOnGithubActions) + $(_ShouldSplit) + $(MSBuildProjectFullPath) + + + + diff --git a/tests/Shared/GetTestProjects.proj b/tests/Shared/GetTestProjects.proj index 6fd3a457e40..33521d0019b 100644 --- a/tests/Shared/GetTestProjects.proj +++ b/tests/Shared/GetTestProjects.proj @@ -13,7 +13,7 @@ + DependsOnTargets="EnumerateSplitTests;GenerateSplitTestsMatrix"> @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - + + + + <_BuildOs Condition="'$(BuildOs)' == ''">linux + <_BuildOs Condition="'$(BuildOs)' != ''">$(BuildOs) + + + + <_TestProjectsToExclude Include="$(RepoRoot)tests\Shared\**\*Tests.csproj" /> + <_TestProjectsToExclude Include="$(RepoRoot)tests\testproject\**\*Tests.csproj" /> + <_TestProjectsToExclude Include="$(RepoRoot)tests\TestingAppHost1\**\*Tests.csproj" /> + + + <_TestProjectsToExclude Include="$(RepoRoot)tests\Aspire.EndToEnd.Tests\**\*Tests.csproj" /> + + <_TestProjectsToExclude Include="$(RepoRoot)tests\Aspire.Templates.Tests\**\*Tests.csproj" /> + + <_TestProjects Include="$(RepoRoot)tests\**\*Tests.csproj" + Exclude="@(_TestProjectsToExclude)" /> + + + + + + + + + <_ProjectsForSplitCheck Remove="@(_ProjectsForSplitCheck)" Condition="'%(RunTestsOnGithubActions)' != 'true' or '%(SplitTests)' != 'true'" /> + + + + + + + + @@ -65,9 +105,11 @@ <_BuildOs Condition="'$(BuildOs)' == ''">unknown <_BuildOs Condition="'$(BuildOs)' != ''">$(BuildOs) - + + + - + <_PwshCommand>pwsh @@ -79,10 +121,9 @@ <_MatrixCommand>$(_MatrixCommand) -BuildOs "$(_BuildOs)" - + From a616cb129b77049f2ef9b7f1d651e0ab224f0de8 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Thu, 16 Oct 2025 21:22:23 -0400 Subject: [PATCH 09/48] wip --- eng/Testing.targets | 28 +++++++++++++++++++++++----- tests/Directory.Build.targets | 31 +++++++------------------------ tests/Shared/GetTestProjects.proj | 17 ++++++++++------- 3 files changed, 40 insertions(+), 36 deletions(-) diff --git a/eng/Testing.targets b/eng/Testing.targets index 4530ea59993..684b6d9fa25 100644 --- a/eng/Testing.targets +++ b/eng/Testing.targets @@ -120,11 +120,29 @@ - <_ProjectInfo Include="$(MSBuildProjectFullPath)"> - $(RunOnGithubActions) - $(_ShouldSplit) - $(MSBuildProjectFullPath) - + + <_ProjectInfo Condition="'$(BuildOs)' == 'windows'" + Include="$(MSBuildProjectFullPath)" + RunTestsOnGithubActions="$(RunOnGithubActionsWindows)" + SplitTests="$(_ShouldSplit)" + MSBuildSourceProjectFile="$(MSBuildProjectFullPath)" /> + <_ProjectInfo Condition="'$(BuildOs)' == 'linux'" + Include="$(MSBuildProjectFullPath)" + RunTestsOnGithubActions="$(RunOnGithubActionsLinux)" + SplitTests="$(_ShouldSplit)" + MSBuildSourceProjectFile="$(MSBuildProjectFullPath)" /> + <_ProjectInfo Condition="'$(BuildOs)' == 'darwin'" + Include="$(MSBuildProjectFullPath)" + RunTestsOnGithubActions="$(RunOnGithubActionsMacOS)" + SplitTests="$(_ShouldSplit)" + MSBuildSourceProjectFile="$(MSBuildProjectFullPath)" /> + + + <_ProjectInfo Condition="'$(BuildOs)' == ''" + Include="$(MSBuildProjectFullPath)" + RunTestsOnGithubActions="$(RunOnGithubActions)" + SplitTests="$(_ShouldSplit)" + MSBuildSourceProjectFile="$(MSBuildProjectFullPath)" /> diff --git a/tests/Directory.Build.targets b/tests/Directory.Build.targets index 01d6925f518..8a9687f856a 100644 --- a/tests/Directory.Build.targets +++ b/tests/Directory.Build.targets @@ -79,7 +79,7 @@ - @@ -88,15 +88,15 @@ <_DiscoveryScriptPath>$(RepoRoot)eng\scripts\extract-test-metadata.ps1 - + <_TestListFile>$(TestArchiveTestsDir)$(MSBuildProjectName).tests.list <_MetadataFile>$(TestArchiveTestsDir)$(MSBuildProjectName).tests.metadata.json - + <_RelativeProjectPath>$(MSBuildProjectDirectory.Replace('$(RepoRoot)', '')) <_RelativeProjectPath>$(_RelativeProjectPath.Replace('\', '/')) - + <_CollectionsToSkip Condition="'$(TestCollectionsToSkipSplitting)' != ''">$(TestCollectionsToSkipSplitting) <_CollectionsToSkip Condition="'$(TestCollectionsToSkipSplitting)' == ''"> @@ -136,7 +136,7 @@ - + <_PwshCommand>pwsh <_PwshCommand Condition="'$(OS)' == 'Windows_NT'">powershell @@ -151,8 +151,8 @@ <_DiscoveryCommand>$(_DiscoveryCommand) -RepoRoot "$(RepoRoot)" - @@ -166,23 +166,6 @@ - - - - - - - - diff --git a/tests/Shared/GetTestProjects.proj b/tests/Shared/GetTestProjects.proj index 33521d0019b..68765689d12 100644 --- a/tests/Shared/GetTestProjects.proj +++ b/tests/Shared/GetTestProjects.proj @@ -17,6 +17,11 @@ + + <_BuildOs Condition="'$(BuildOs)' == ''">linux + <_BuildOs Condition="'$(BuildOs)' != ''">$(BuildOs) + + <_TestProjectsToExclude Include="$(RepoRoot)tests\Shared\**\*Tests.csproj" /> <_TestProjectsToExclude Include="$(RepoRoot)tests\testproject\**\*Tests.csproj" /> @@ -24,15 +29,15 @@ <_TestProjectsToExclude Include="$(RepoRoot)tests\Aspire.EndToEnd.Tests\**\*Tests.csproj" /> - - <_TestProjectsToExclude Include="$(RepoRoot)tests\Aspire.Templates.Tests\**\*Tests.csproj" /> <_TestProjects Include="$(RepoRoot)tests\**\*Tests.csproj" Exclude="@(_TestProjectsToExclude)" /> - + @@ -51,9 +56,9 @@ Lines="@(RegularTestProjects->'%(ShortName)')" Overwrite="true" /> - + @@ -71,8 +76,6 @@ <_TestProjectsToExclude Include="$(RepoRoot)tests\Aspire.EndToEnd.Tests\**\*Tests.csproj" /> - - <_TestProjectsToExclude Include="$(RepoRoot)tests\Aspire.Templates.Tests\**\*Tests.csproj" /> <_TestProjects Include="$(RepoRoot)tests\**\*Tests.csproj" Exclude="@(_TestProjectsToExclude)" /> From 35378a27d734260c2328097b7aebcdeb1fb04f80 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Thu, 16 Oct 2025 21:56:46 -0400 Subject: [PATCH 10/48] restore --- .github/actions/enumerate-tests/action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/actions/enumerate-tests/action.yml b/.github/actions/enumerate-tests/action.yml index c132f9bd518..888e21cf229 100644 --- a/.github/actions/enumerate-tests/action.yml +++ b/.github/actions/enumerate-tests/action.yml @@ -31,6 +31,10 @@ runs: with: global-json-file: ${{ github.workspace }}/global.json + - name: Restore + shell: bash + run: ./restore.sh + - name: Get list of integration tests if: ${{ inputs.includeIntegrations }} shell: pwsh From f6e39408a77c0015b9aca7c77169f423655a78ae Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Thu, 16 Oct 2025 22:11:38 -0400 Subject: [PATCH 11/48] wip --- eng/scripts/generate-test-matrix.ps1 | 31 ++++++++++++++++++++++++++-- tests/Shared/GetTestProjects.proj | 2 ++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/eng/scripts/generate-test-matrix.ps1 b/eng/scripts/generate-test-matrix.ps1 index 06781a271e5..405dd8d83b7 100644 --- a/eng/scripts/generate-test-matrix.ps1 +++ b/eng/scripts/generate-test-matrix.ps1 @@ -43,7 +43,9 @@ param( [string]$OutputDirectory, [Parameter(Mandatory=$false)] [ValidateSet('windows','linux','darwin','')] - [string]$BuildOs = '' + [string]$BuildOs = '', + [Parameter(Mandatory=$false)] + [string]$RegularTestProjectsFile = '' ) $ErrorActionPreference = 'Stop' @@ -132,6 +134,22 @@ function New-EntryClass($full,$meta) { } } +function New-EntryRegular($shortName) { + [ordered]@{ + type = 'regular' + projectName = "Aspire.$shortName.Tests" + name = $shortName + shortname = $shortName + testProjectPath = "tests/Aspire.$shortName.Tests/Aspire.$shortName.Tests.csproj" + filterArg = "" + requiresNugets = $false + requiresTestSdk = $false + enablePlaywrightInstall = $false + testSessionTimeout = '20m' + testHangTimeout = '10m' + } +} + if (-not (Test-Path $TestListsDirectory)) { Write-Warning "Test lists directory not found: $TestListsDirectory" exit 0 @@ -180,7 +198,16 @@ foreach ($lf in $listFiles) { } } +# Add regular (non-split) test projects if provided +if ($RegularTestProjectsFile -and (Test-Path $RegularTestProjectsFile)) { + $regularProjects = @(Get-Content $RegularTestProjectsFile | Where-Object { $_ -and -not [string]::IsNullOrWhiteSpace($_) }) + Write-Host "Adding $($regularProjects.Count) regular test project(s)" + foreach ($shortName in $regularProjects) { + $entries.Add( (New-EntryRegular $shortName) ) | Out-Null + } +} + $matrix = @{ include = $entries } New-Item -ItemType Directory -Force -Path $OutputDirectory | Out-Null $matrix | ConvertTo-Json -Depth 10 -Compress | Set-Content -Path (Join-Path $OutputDirectory 'split-tests-matrix.json') -Encoding UTF8 -Write-Host "Matrix entries: $($entries.Count)" \ No newline at end of file +Write-Host "Matrix entries: $($entries.Count)" diff --git a/tests/Shared/GetTestProjects.proj b/tests/Shared/GetTestProjects.proj index 68765689d12..7370dea356c 100644 --- a/tests/Shared/GetTestProjects.proj +++ b/tests/Shared/GetTestProjects.proj @@ -118,10 +118,12 @@ <_PwshCommand>pwsh <_PwshCommand Condition="'$(OS)' == 'Windows_NT'">powershell <_MatrixScriptPath>$([MSBuild]::NormalizePath($(RepoRoot), 'eng', 'scripts', 'generate-test-matrix.ps1')) + <_RegularTestProjectsFile>$([MSBuild]::NormalizePath($(MSBuildProjectDirectory), $(TestsListOutputPath))) <_MatrixCommand>$(_PwshCommand) -NoProfile -ExecutionPolicy Bypass -File "$(_MatrixScriptPath)" <_MatrixCommand>$(_MatrixCommand) -TestListsDirectory "$(_TestListsDir)" <_MatrixCommand>$(_MatrixCommand) -OutputDirectory "$(TestMatrixOutputPath)" <_MatrixCommand>$(_MatrixCommand) -BuildOs "$(_BuildOs)" + <_MatrixCommand>$(_MatrixCommand) -RegularTestProjectsFile "$(_RegularTestProjectsFile)" Date: Fri, 17 Oct 2025 00:27:31 -0400 Subject: [PATCH 12/48] fixing --- eng/AfterSolutionBuild.targets | 28 +++ eng/TestEnumerationRunsheetBuilder/DESIGN.md | 166 ++++++++++++++++++ .../TestEnumerationRunsheetBuilder.targets | 94 ++++++++++ eng/scripts/process-test-enumeration.ps1 | 145 +++++++++++++++ tools/ExtractTestPartitions/Program.cs | 8 +- 5 files changed, 437 insertions(+), 4 deletions(-) create mode 100644 eng/TestEnumerationRunsheetBuilder/DESIGN.md create mode 100644 eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets create mode 100644 eng/scripts/process-test-enumeration.ps1 diff --git a/eng/AfterSolutionBuild.targets b/eng/AfterSolutionBuild.targets index 3288a591e3d..943f173352b 100644 --- a/eng/AfterSolutionBuild.targets +++ b/eng/AfterSolutionBuild.targets @@ -105,4 +105,32 @@ + + + + + + + + + <_BuildOs Condition="'$(BuildOs)' == ''">linux + <_BuildOs Condition="'$(BuildOs)' != ''">$(BuildOs) + <_TestsListOutputPath Condition="'$(TestsListOutputPath)' != ''">$([MSBuild]::NormalizePath('$(RepoRoot)', '$(TestsListOutputPath)')) + <_TestsListOutputPath Condition="'$(TestsListOutputPath)' == ''">$(ArtifactsDir)/TestsForGithubActions.list + <_TestMatrixOutputPath Condition="'$(TestMatrixOutputPath)' != ''">$([MSBuild]::NormalizePath('$(RepoRoot)', '$(TestMatrixOutputPath)')) + <_ProcessScript>$([MSBuild]::NormalizePath($(RepoRoot), 'eng', 'scripts', 'process-test-enumeration.ps1')) + + + + + + + + diff --git a/eng/TestEnumerationRunsheetBuilder/DESIGN.md b/eng/TestEnumerationRunsheetBuilder/DESIGN.md new file mode 100644 index 00000000000..9aae4375461 --- /dev/null +++ b/eng/TestEnumerationRunsheetBuilder/DESIGN.md @@ -0,0 +1,166 @@ +# TestEnumerationRunsheetBuilder Design + +## Overview + +This document describes the design for migrating the current `GetTestProjects.proj` test enumeration mechanism to work through the Arcade SDK's runsheet builder pattern. + +## Current Architecture + +### GetTestProjects.proj (Current) +- **Approach**: Centralized test project discovery and enumeration +- **Invocation**: Manual execution via `dotnet build tests/Shared/GetTestProjects.proj` +- **Process**: + 1. Discovers all test projects using glob patterns + 2. Calls MSBuild on each project to determine GitHub Actions eligibility + 3. Builds split test projects to generate test class lists + 4. Generates final test lists and matrices using PowerShell scripts + +### Problems with Current Approach +- Requires explicit invocation outside the standard build process +- Not integrated with Arcade SDK's runsheet builder mechanism +- Duplicates logic that could be shared with other runsheet builders + +## New Architecture: TestEnumerationRunsheetBuilder + +### Design Principles +1. **Distributed Processing**: Each test project generates its own enumeration data during build +2. **Arcade SDK Integration**: Follows the same pattern as existing runsheet builders +3. **Reuse Existing Logic**: Leverages existing test enumeration and splitting mechanisms +4. **Centralized Combination**: Final aggregation happens in `AfterSolutionBuild.targets` + +### Components + +#### 1. TestEnumerationRunsheetBuilder.targets +- **Location**: `eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets` +- **Purpose**: Runs once per test project to generate test enumeration data +- **Outputs**: Per-project test enumeration files in `ArtifactsTmpDir` + +#### 2. Enhanced AfterSolutionBuild.targets +- **Purpose**: Combines individual test enumeration files into final outputs +- **Trigger**: When `TestRunnerName=TestEnumerationRunsheetBuilder` +- **Outputs**: Same as current GetTestProjects.proj (test lists and matrices) + +### Flow Diagram + +``` +Build Process +├── For each test project: +│ ├── TestEnumerationRunsheetBuilder.targets runs +│ ├── Generates project-specific enumeration data +│ └── Writes to artifacts/tmp/{project}.testenumeration.json +│ +└── After all projects built: + ├── AfterSolutionBuild.targets runs + ├── Collects all testenumeration.json files + ├── Processes split tests (if any) + └── Generates final outputs: + ├── TestsForGithubActions.list + ├── TestsForGithubActions.list.split-projects + └── test-matrices/*.json +``` + +## Implementation Details + +### TestEnumerationRunsheetBuilder.targets + +```msbuild + + + + + + + + + + <_EnumerationData Include="{ + 'project': '$(MSBuildProjectName)', + 'fullPath': '$(MSBuildProjectFullPath)', + 'shortName': '$(_ShortName)', + 'runOnGithubActions': '%(_ProjectInfo.RunTestsOnGithubActions)', + 'splitTests': '%(_ProjectInfo.SplitTests)' + }" /> + + + + + +``` + +### Enhanced AfterSolutionBuild.targets + +```msbuild + + + + + <_TestEnumerationFiles Include="$(ArtifactsTmpDir)/*.testenumeration.json" /> + + + + + <_ProcessingScript> + # PowerShell script to: + # 1. Read all testenumeration.json files + # 2. Filter by OS and eligibility + # 3. Generate test lists and split test lists + # 4. Call existing matrix generation script for split tests + + + + + + +``` + +## Migration Strategy + +### Phase 1: Implementation +1. Create `TestEnumerationRunsheetBuilder.targets` +2. Enhance `AfterSolutionBuild.targets` with test enumeration logic +3. Implement PowerShell processing script + +### Phase 2: Integration +1. Update GitHub Actions workflows to use new approach +2. Test compatibility with existing split test functionality +3. Validate output format matches current GetTestProjects.proj + +### Phase 3: Cleanup +1. Deprecate GetTestProjects.proj usage in workflows +2. Remove manual invocation commands +3. Document new usage pattern + +## Usage + +### Command Line +```bash +# Instead of manual GetTestProjects.proj invocation: +dotnet build tests/Shared/GetTestProjects.proj /bl:artifacts/log/Debug/GetTestProjects.binlog /p:TestsListOutputPath=artifacts/TestsForGithubActions.list /p:TestMatrixOutputPath=artifacts/test-matrices/ /p:ContinuousIntegrationBuild=true /p:BuildOs=linux + +# New approach using runsheet builder: +./build.cmd -test /p:TestRunnerName=TestEnumerationRunsheetBuilder /p:TestsListOutputPath=artifacts/TestsForGithubActions.list /p:TestMatrixOutputPath=artifacts/test-matrices/ /p:ContinuousIntegrationBuild=true /p:BuildOs=linux +``` + +### Integration with CI +The new approach integrates seamlessly with the existing build infrastructure and requires minimal changes to GitHub Actions workflows. + +## Benefits + +1. **Consistency**: Follows the same pattern as other runsheet builders +2. **Automatic Discovery**: No manual project enumeration required +3. **Build Integration**: Leverages existing build process and caching +4. **Maintainability**: Reduces code duplication and improves consistency +5. **Extensibility**: Easy to add new test enumeration features + +## Backward Compatibility + +- Existing GetTestProjects.proj functionality remains unchanged +- New approach generates identical output formats +- Migration can be done incrementally per workflow \ No newline at end of file diff --git a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets new file mode 100644 index 00000000000..7778cf3ea63 --- /dev/null +++ b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets @@ -0,0 +1,94 @@ + + + + + + + + <_ShouldSkipProject>false + <_ShouldSkipProject Condition="$(MSBuildProjectDirectory.Contains('tests\Shared'))">true + <_ShouldSkipProject Condition="$(MSBuildProjectDirectory.Contains('tests\testproject'))">true + <_ShouldSkipProject Condition="$(MSBuildProjectDirectory.Contains('tests\TestingAppHost1'))">true + <_ShouldSkipProject Condition="'$(MSBuildProjectName)' == 'Aspire.EndToEnd.Tests'">true + + + <_ShortName>$([System.IO.Path]::GetFileNameWithoutExtension('$(MSBuildProjectName)').Replace('Aspire.', '').Replace('.Tests', '')) + + + + + + <_CurrentProject Include="$(MSBuildProjectFullPath)" /> + + + + + + + + + + + + + <_RelativeProjectPath>$([System.String]::Copy('$(MSBuildProjectFullPath)').Replace('$(RepoRoot)', '')) + + + <_HelixDir>$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix')) + <_TestListFile>$(_HelixDir)$(MSBuildProjectName).tests.list + <_MetadataFile>$(_HelixDir)$(MSBuildProjectName).tests.metadata.json + <_HasTestMetadata Condition="'%(_ProjectInfo.SplitTests)' == 'true' and Exists('$(_TestListFile)') and Exists('$(_MetadataFile)')">true + <_HasTestMetadata Condition="'$(_HasTestMetadata)' != 'true'">false + + + <_EnumerationJson>{ + "project": "$(MSBuildProjectName)", + "fullPath": "$(_RelativeProjectPath)", + "shortName": "$(_ShortName)", + "runOnGithubActions": "%(_ProjectInfo.RunTestsOnGithubActions)", + "splitTests": "%(_ProjectInfo.SplitTests)", + "buildOs": "$(BuildOs)", + "hasTestMetadata": "$(_HasTestMetadata)", + "testListFile": "$(_TestListFile.Replace('$(RepoRoot)', ''))", + "metadataFile": "$(_MetadataFile.Replace('$(RepoRoot)', ''))" +} + + + + + + + + + + + + + + \ No newline at end of file diff --git a/eng/scripts/process-test-enumeration.ps1 b/eng/scripts/process-test-enumeration.ps1 new file mode 100644 index 00000000000..ef1830406b8 --- /dev/null +++ b/eng/scripts/process-test-enumeration.ps1 @@ -0,0 +1,145 @@ +#!/usr/bin/env pwsh + +param( + [Parameter(Mandatory=$true)] + [string]$BuildOs, + + [Parameter(Mandatory=$true)] + [string]$TestsListOutputPath, + + [Parameter(Mandatory=$false)] + [string]$TestMatrixOutputPath, + + [Parameter(Mandatory=$true)] + [string]$ArtifactsTmpDir, + + [Parameter(Mandatory=$true)] + [string]$RepoRoot +) + +Write-Host "Processing test enumeration files for BuildOs: $BuildOs" +Write-Host "TestsListOutputPath: $TestsListOutputPath" +Write-Host "TestMatrixOutputPath: $TestMatrixOutputPath" +Write-Host "ArtifactsTmpDir: $ArtifactsTmpDir" + +# Find all test enumeration files +$enumerationFiles = Get-ChildItem -Path $ArtifactsTmpDir -Filter '*.testenumeration.json' -ErrorAction SilentlyContinue + +if (-not $enumerationFiles) { + Write-Host "No test enumeration files found in $ArtifactsTmpDir" + # Create empty output files + "" | Set-Content $TestsListOutputPath + if ($TestMatrixOutputPath) { + New-Item -Path $TestMatrixOutputPath -ItemType Directory -Force | Out-Null + } + exit 0 +} + +Write-Host "Found $($enumerationFiles.Count) test enumeration files" + +# Process enumeration files +$regularTestProjects = @() +$splitTestProjects = @() + +foreach ($file in $enumerationFiles) { + try { + $content = Get-Content -Raw $file.FullName | ConvertFrom-Json + + # Filter by BuildOs and eligibility + if ($content.buildOs -eq $BuildOs -and $content.runOnGithubActions -eq 'true') { + if ($content.splitTests -eq 'true') { + $splitTestProjects += $content.shortName + } else { + $regularTestProjects += $content.shortName + } + Write-Host " Included: $($content.shortName) (Split: $($content.splitTests))" + } else { + Write-Host " Excluded: $($content.shortName) (BuildOs: $($content.buildOs), RunOnGithubActions: $($content.runOnGithubActions))" + } + } + catch { + Write-Warning "Failed to process $($file.FullName): $_" + } +} + +Write-Host "Regular test projects: $($regularTestProjects.Count)" +Write-Host "Split test projects: $($splitTestProjects.Count)" + +# Create output directory if needed +$outputDir = Split-Path $TestsListOutputPath -Parent +if (-not (Test-Path $outputDir)) { + New-Item -Path $outputDir -ItemType Directory -Force | Out-Null +} + +# Write regular test projects list +$regularTestProjects | Set-Content $TestsListOutputPath + +# Write split test projects list if any exist +if ($splitTestProjects.Count -gt 0) { + $splitTestProjects | Select-Object -Unique | Set-Content "$TestsListOutputPath.split-projects" + Write-Host "Split projects written to: $TestsListOutputPath.split-projects" +} +else { + Write-Host "No split test projects found, skipping split-projects file creation" +} + + +# Generate test matrices if output path is specified +if ($TestMatrixOutputPath) { + Write-Host "Generating test matrices..." + + # Check if TestMatrixOutputPath ends with .json (single file) or is a directory + $isJsonFile = $TestMatrixOutputPath -match '\.json$' + + if ($isJsonFile) { + # Single JSON file output - create directory for intermediate files + $tempMatrixDir = Join-Path (Split-Path $TestMatrixOutputPath -Parent) 'temp-matrix' + New-Item -Path $tempMatrixDir -ItemType Directory -Force | Out-Null + + # Call existing matrix generation script if split tests exist + if ($splitTestProjects.Count -gt 0) { + $matrixScriptPath = Join-Path $RepoRoot 'eng/scripts/generate-test-matrix.ps1' + $testListsDir = Join-Path (Split-Path $TestsListOutputPath -Parent) 'helix' + Write-Host "Calling matrix generation script..." + & $matrixScriptPath -TestListsDirectory $testListsDir -OutputDirectory $tempMatrixDir -BuildOs $BuildOs -RegularTestProjectsFile $TestsListOutputPath + + # Copy the generated matrix file to the expected location + $generatedMatrixFile = Join-Path $tempMatrixDir 'split-tests-matrix.json' + if (Test-Path $generatedMatrixFile) { + Copy-Item $generatedMatrixFile $TestMatrixOutputPath + Write-Host "Matrix file copied to: $TestMatrixOutputPath" + } else { + Write-Warning "Expected matrix file not found at: $generatedMatrixFile" + } + + # Clean up temporary directory + Remove-Item $tempMatrixDir -Recurse -Force -ErrorAction SilentlyContinue + } else { + # No split tests, create empty matrix + '{"include":[]}' | Set-Content $TestMatrixOutputPath + Write-Host "No split tests found, created empty matrix at: $TestMatrixOutputPath" + } + } else { + # Directory output (original behavior) + New-Item -Path $TestMatrixOutputPath -ItemType Directory -Force | Out-Null + + # Call existing matrix generation script if split tests exist + if ($splitTestProjects.Count -gt 0) { + $matrixScriptPath = Join-Path $RepoRoot 'eng/scripts/generate-test-matrix.ps1' + $testListsDir = Join-Path (Split-Path $TestsListOutputPath -Parent) 'helix' + Write-Host "Calling matrix generation script..." + & $matrixScriptPath -TestListsDirectory $testListsDir -OutputDirectory $TestMatrixOutputPath -BuildOs $BuildOs -RegularTestProjectsFile $TestsListOutputPath + } + } +} else { + Write-Host "No TestMatrixOutputPath specified, skipping matrix generation" +} + +Write-Host "Test enumeration processing completed" +Write-Host "Regular projects written to: $TestsListOutputPath" +#if ($splitTestProjects.Count -gt 0) { + #Write-Host "Split projects written to: $TestsListOutputPath.split-projects" +#} +if ($TestMatrixOutputPath) { + Write-Host "Test matrices written to: $TestMatrixOutputPath" +} diff --git a/tools/ExtractTestPartitions/Program.cs b/tools/ExtractTestPartitions/Program.cs index b559007c09a..8449a0d45ab 100644 --- a/tools/ExtractTestPartitions/Program.cs +++ b/tools/ExtractTestPartitions/Program.cs @@ -55,14 +55,14 @@ static void ExtractPartitions(string assemblyPath, string outputFile) // Some types couldn't be loaded due to missing dependencies // Use the types that did load types = ex.Types.Where(t => t != null).ToArray()!; - Console.WriteLine($"Warning: Some types could not be loaded. Loaded {types.Length} types successfully."); + Console.WriteLine($"** Some types could not be loaded. Loaded {types.Length} types successfully."); } foreach (var type in types) { // Check if type has Collection or Trait attributes var attributes = type.GetCustomAttributesData(); - + foreach (var attr in attributes) { var attrTypeName = attr.AttributeType.FullName ?? attr.AttributeType.Name; @@ -86,8 +86,8 @@ static void ExtractPartitions(string assemblyPath, string outputFile) { var key = attr.ConstructorArguments[0].Value as string; var value = attr.ConstructorArguments[1].Value as string; - - if (key?.Equals("Partition", StringComparison.OrdinalIgnoreCase) == true && + + if (key?.Equals("Partition", StringComparison.OrdinalIgnoreCase) == true && !string.IsNullOrWhiteSpace(value)) { partitions.Add(value); From cca33f51ee8b877d12d36a4dcc8dc3686909db37 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 00:37:07 -0400 Subject: [PATCH 13/48] fixy simplify --- .github/actions/enumerate-tests/action.yml | 120 +++++------- .github/workflows/tests.yml | 209 +++++++-------------- 2 files changed, 111 insertions(+), 218 deletions(-) diff --git a/.github/actions/enumerate-tests/action.yml b/.github/actions/enumerate-tests/action.yml index 888e21cf229..fcdc6f4f69f 100644 --- a/.github/actions/enumerate-tests/action.yml +++ b/.github/actions/enumerate-tests/action.yml @@ -11,15 +11,12 @@ inputs: default: false outputs: - integrations_tests_matrix: - description: Integration tests matrix - value: ${{ steps.generate_integrations_matrix.outputs.integrations_tests_matrix }} - templates_tests_matrix: - description: Templates tests matrix - value: ${{ steps.generate_templates_matrix.outputs.templates_tests_matrix }} - split_tests_matrix: - description: Split tests matrix (for collection/class-based splitting) - value: ${{ steps.generate_split_tests_matrix.outputs.split_tests_matrix }} + tests_matrix_requires_nugets: + description: Combined tests matrix for tests that require nugets + value: ${{ steps.generate_combined_matrix.outputs.tests_matrix_requires_nugets }} + tests_matrix_no_nugets: + description: Combined tests matrix for tests that do not require nugets + value: ${{ steps.generate_combined_matrix.outputs.tests_matrix_no_nugets }} runs: using: "composite" steps: @@ -35,83 +32,56 @@ runs: shell: bash run: ./restore.sh - - name: Get list of integration tests - if: ${{ inputs.includeIntegrations }} - shell: pwsh + - name: Generate combined test matrix + shell: bash run: > - dotnet build ${{ github.workspace }}/tests/Shared/GetTestProjects.proj - /bl:${{ github.workspace }}/artifacts/log/Debug/GetTestProjects.binlog - /p:TestsListOutputPath=${{ github.workspace }}/artifacts/TestsForGithubActions.list - /p:TestMatrixOutputPath=${{ github.workspace }}/artifacts/test-matrices/ - /p:ContinuousIntegrationBuild=true + ./build.sh -test + /p:TestRunnerName=TestEnumerationRunsheetBuilder + /p:TestsListOutputPath=artifacts/TestsForGithubActions.list + /p:TestMatrixOutputPath=artifacts/combined-test-matrix.json /p:BuildOs=${{ runner.os == 'Linux' && 'linux' || runner.os == 'macOS' && 'darwin' || 'windows' }} + -p:PrepareForHelix=true + -bl - - name: Generate list of template tests - if: ${{ inputs.includeTemplates }} - shell: pwsh - run: > - dotnet build ${{ github.workspace }}/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj - "/t:Build;ExtractTestClassNames" - /bl:${{ github.workspace }}/artifacts/log/Debug/BuildTemplatesTests.binlog - -p:ExtractTestClassNamesForHelix=true - -p:PrepareForHelix=true - -p:ExtractTestClassNamesPrefix=Aspire.Templates.Tests - -p:InstallBrowsersForPlaywright=false - - - name: Generate tests matrix - id: generate_integrations_matrix - if: ${{ inputs.includeIntegrations }} + - name: Generate combined matrix outputs + id: generate_combined_matrix shell: pwsh run: | - $filePath = "${{ github.workspace }}/artifacts/TestsForGithubActions.list" - $lines = Get-Content $filePath - $jsonObject = @{ - "shortname" = $lines | Sort-Object - } - $jsonString = ConvertTo-Json $jsonObject -Compress - "integrations_tests_matrix=$jsonString" - "integrations_tests_matrix=$jsonString" | Out-File -FilePath $env:GITHUB_OUTPUT + $matrixFilePath = "${{ github.workspace }}/artifacts/combined-test-matrix.json" + if (Test-Path $matrixFilePath) { + $matrixContent = Get-Content -Raw $matrixFilePath | ConvertFrom-Json - - name: Generate templates matrix - id: generate_templates_matrix - if: ${{ inputs.includeTemplates }} - shell: pwsh - run: | - $inputFilePath = "${{ github.workspace }}/artifacts/helix/templates-tests/Aspire.Templates.Tests.tests.list" - $lines = Get-Content $inputFilePath + # Split tests based on requiresNugets property + $testsRequiringNugets = @() + $testsNotRequiringNugets = @() - $prefix = "Aspire.Templates.Tests." - $lines = Get-Content $inputFilePath | ForEach-Object { - # Strip "class:" or "collection:" prefix first - $line = $_ -replace "^(class|collection|uncollected):", "" - # Then strip the test class prefix - $line -replace "^$prefix", "" - } + foreach ($test in $matrixContent.include) { + if ($test.requiresNugets -eq $true) { + $testsRequiringNugets += $test + } else { + $testsNotRequiringNugets += $test + } + } - $jsonObject = @{ - "shortname" = $lines | Sort-Object - } - $jsonString = ConvertTo-Json $jsonObject -Compress - "templates_tests_matrix=$jsonString" - "templates_tests_matrix=$jsonString" | Out-File -FilePath $env:GITHUB_OUTPUT + # Create matrices + $nugetMatrix = @{ "include" = $testsRequiringNugets } + $nonNugetMatrix = @{ "include" = $testsNotRequiringNugets } - - name: Generate split tests matrix - id: generate_split_tests_matrix - if: ${{ inputs.includeIntegrations }} - shell: pwsh - run: | - $matrixFilePath = "${{ github.workspace }}/artifacts/test-matrices/split-tests-matrix.json" - if (Test-Path $matrixFilePath) { - $matrixContent = Get-Content -Raw $matrixFilePath - # Validate it's valid JSON - $null = $matrixContent | ConvertFrom-Json - "split_tests_matrix=$matrixContent" | Out-File -FilePath $env:GITHUB_OUTPUT - Write-Host "Split tests matrix generated successfully" + $nugetMatrixJson = ConvertTo-Json $nugetMatrix -Compress -Depth 10 + $nonNugetMatrixJson = ConvertTo-Json $nonNugetMatrix -Compress -Depth 10 + + "tests_matrix_requires_nugets=$nugetMatrixJson" | Out-File -FilePath $env:GITHUB_OUTPUT + "tests_matrix_no_nugets=$nonNugetMatrixJson" | Out-File -FilePath $env:GITHUB_OUTPUT + + Write-Host "Combined test matrices generated successfully" + Write-Host "Tests requiring nugets: $($testsRequiringNugets.Count)" + Write-Host "Tests not requiring nugets: $($testsNotRequiringNugets.Count)" } else { - # Empty matrix if no split tests + # Empty matrices if no combined matrix found $emptyMatrix = '{"include":[]}' - "split_tests_matrix=$emptyMatrix" | Out-File -FilePath $env:GITHUB_OUTPUT - Write-Host "No split tests matrix found, using empty matrix" + "tests_matrix_requires_nugets=$emptyMatrix" | Out-File -FilePath $env:GITHUB_OUTPUT + "tests_matrix_no_nugets=$emptyMatrix" | Out-File -FilePath $env:GITHUB_OUTPUT + Write-Host "No combined test matrix found, using empty matrices" } - name: Upload logs diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5e4e91b6925..3dfeceb62f1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,48 +9,12 @@ on: type: string jobs: - # Duplicated jobs so their dependencies are not blocked on both the - # setup jobs - - setup_for_tests_lin: - name: Setup for tests (Linux) + setup_for_tests: + name: Setup for tests runs-on: ubuntu-latest outputs: - integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }} - templates_tests_matrix: ${{ steps.generate_tests_matrix.outputs.templates_tests_matrix }} - split_tests_matrix: ${{ steps.generate_tests_matrix.outputs.split_tests_matrix }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - uses: ./.github/actions/enumerate-tests - id: generate_tests_matrix - with: - includeIntegrations: true - includeTemplates: true - - setup_for_tests_macos: - name: Setup for tests (macOS) - runs-on: macos-latest - outputs: - integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }} - templates_tests_matrix: ${{ steps.generate_tests_matrix.outputs.templates_tests_matrix }} - split_tests_matrix: ${{ steps.generate_tests_matrix.outputs.split_tests_matrix }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - uses: ./.github/actions/enumerate-tests - id: generate_tests_matrix - with: - includeIntegrations: true - includeTemplates: true - - setup_for_tests_win: - name: Setup for tests (Windows) - runs-on: windows-latest - outputs: - integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }} - templates_tests_matrix: ${{ steps.generate_tests_matrix.outputs.templates_tests_matrix }} - split_tests_matrix: ${{ steps.generate_tests_matrix.outputs.split_tests_matrix }} + tests_matrix_requires_nugets: ${{ steps.generate_tests_matrix.outputs.tests_matrix_requires_nugets }} + tests_matrix_no_nugets: ${{ steps.generate_tests_matrix.outputs.tests_matrix_no_nugets }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -66,159 +30,121 @@ jobs: with: versionOverrideArg: ${{ inputs.versionOverrideArg }} - integrations_test_lin: + tests_no_nugets_lin: uses: ./.github/workflows/run-tests.yml - name: Integrations Linux - needs: setup_for_tests_lin + name: Tests Linux (No Nugets) + needs: setup_for_tests + if: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets).include[0] != null }} strategy: fail-fast: false - matrix: - ${{ fromJson(needs.setup_for_tests_lin.outputs.integrations_tests_matrix) }} + matrix: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets) }} with: testShortName: ${{ matrix.shortname }} os: "ubuntu-latest" - # Docker tests are run on linux, and Hosting tests take longer to finish - testSessionTimeout: ${{ matrix.shortname == 'Hosting' && '25m' || '15m' }} - extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\"" + testProjectPath: ${{ matrix.testProjectPath }} + testSessionTimeout: ${{ matrix.testSessionTimeout }} + testHangTimeout: ${{ matrix.testHangTimeout }} + extraTestArgs: "${{ matrix.extraTestArgs }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} + requiresNugets: ${{ matrix.requiresNugets }} + requiresTestSdk: ${{ matrix.requiresTestSdk }} + enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} - integrations_test_macos: + tests_no_nugets_macos: uses: ./.github/workflows/run-tests.yml - name: Integrations macos - needs: setup_for_tests_macos + name: Tests macOS (No Nugets) + needs: setup_for_tests + if: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets).include[0] != null }} strategy: fail-fast: false - matrix: - ${{ fromJson(needs.setup_for_tests_macos.outputs.integrations_tests_matrix) }} + matrix: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets) }} with: testShortName: ${{ matrix.shortname }} os: "macos-latest" - extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\"" + testProjectPath: ${{ matrix.testProjectPath }} + testSessionTimeout: ${{ matrix.testSessionTimeout }} + testHangTimeout: ${{ matrix.testHangTimeout }} + extraTestArgs: "${{ matrix.extraTestArgs }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} + requiresNugets: ${{ matrix.requiresNugets }} + requiresTestSdk: ${{ matrix.requiresTestSdk }} + enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} - integrations_test_win: + tests_no_nugets_win: uses: ./.github/workflows/run-tests.yml - name: Integrations Windows - needs: setup_for_tests_win + name: Tests Windows (No Nugets) + needs: setup_for_tests + if: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets).include[0] != null }} strategy: fail-fast: false - matrix: - ${{ fromJson(needs.setup_for_tests_win.outputs.integrations_tests_matrix) }} + matrix: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets) }} with: testShortName: ${{ matrix.shortname }} os: "windows-latest" - extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\"" - versionOverrideArg: ${{ inputs.versionOverrideArg }} - - templates_test_lin: - name: Templates Linux - uses: ./.github/workflows/run-tests.yml - needs: [setup_for_tests_lin, build_packages] - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.setup_for_tests_lin.outputs.templates_tests_matrix) }} - with: - testShortName: ${{ matrix.shortname }} - os: "ubuntu-latest" - testProjectPath: tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj - testSessionTimeout: 20m - testHangTimeout: 12m - extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true --filter-class Aspire.Templates.Tests.${{ matrix.shortname }}" - versionOverrideArg: ${{ inputs.versionOverrideArg }} - requiresNugets: true - requiresTestSdk: true - - templates_test_macos: - name: Templates macos - uses: ./.github/workflows/run-tests.yml - needs: [setup_for_tests_macos, build_packages] - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.setup_for_tests_macos.outputs.templates_tests_matrix) }} - with: - testShortName: ${{ matrix.shortname }} - os: "macos-latest" - testProjectPath: tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj - testSessionTimeout: 20m - testHangTimeout: 12m - extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true --filter-class Aspire.Templates.Tests.${{ matrix.shortname }}" + testProjectPath: ${{ matrix.testProjectPath }} + testSessionTimeout: ${{ matrix.testSessionTimeout }} + testHangTimeout: ${{ matrix.testHangTimeout }} + extraTestArgs: "${{ matrix.extraTestArgs }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} - requiresNugets: true - requiresTestSdk: true + requiresNugets: ${{ matrix.requiresNugets }} + requiresTestSdk: ${{ matrix.requiresTestSdk }} + enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} - templates_test_win: - name: Templates Windows + tests_requires_nugets_lin: + name: Tests Linux (Requires Nugets) uses: ./.github/workflows/run-tests.yml - needs: [setup_for_tests_win, build_packages] + needs: [setup_for_tests, build_packages] + if: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets).include[0] != null }} strategy: fail-fast: false - matrix: ${{ fromJson(needs.setup_for_tests_win.outputs.templates_tests_matrix) }} + matrix: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets) }} with: testShortName: ${{ matrix.shortname }} - os: "windows-latest" - testProjectPath: tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj - testSessionTimeout: 20m - testHangTimeout: 12m - extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true --filter-class Aspire.Templates.Tests.${{ matrix.shortname }}" - versionOverrideArg: ${{ inputs.versionOverrideArg }} - requiresNugets: true - requiresTestSdk: true - - split_tests_lin: - name: Split Linux - uses: ./.github/workflows/run-tests.yml - needs: [setup_for_tests_lin, build_packages] - if: ${{ fromJson(needs.setup_for_tests_lin.outputs.split_tests_matrix).include[0] != null }} - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.setup_for_tests_lin.outputs.split_tests_matrix) }} - with: - testShortName: ${{ matrix.projectName }}_${{ matrix.shortname }} os: "ubuntu-latest" testProjectPath: ${{ matrix.testProjectPath }} testSessionTimeout: ${{ matrix.testSessionTimeout }} testHangTimeout: ${{ matrix.testHangTimeout }} - extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true ${{ matrix.filterArg }}" + extraTestArgs: "${{ matrix.extraTestArgs }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} requiresNugets: ${{ matrix.requiresNugets }} requiresTestSdk: ${{ matrix.requiresTestSdk }} enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} - split_tests_macos: - name: Split macos + tests_requires_nugets_macos: + name: Tests macOS (Requires Nugets) uses: ./.github/workflows/run-tests.yml - needs: [setup_for_tests_macos, build_packages] - if: ${{ fromJson(needs.setup_for_tests_macos.outputs.split_tests_matrix).include[0] != null }} + needs: [setup_for_tests, build_packages] + if: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets).include[0] != null }} strategy: fail-fast: false - matrix: ${{ fromJson(needs.setup_for_tests_macos.outputs.split_tests_matrix) }} + matrix: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets) }} with: - testShortName: ${{ matrix.projectName }}_${{ matrix.shortname }} + testShortName: ${{ matrix.shortname }} os: "macos-latest" testProjectPath: ${{ matrix.testProjectPath }} testSessionTimeout: ${{ matrix.testSessionTimeout }} testHangTimeout: ${{ matrix.testHangTimeout }} - extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true ${{ matrix.filterArg }}" + extraTestArgs: "${{ matrix.extraTestArgs }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} requiresNugets: ${{ matrix.requiresNugets }} requiresTestSdk: ${{ matrix.requiresTestSdk }} enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} - split_tests_win: - name: Split Windows + tests_requires_nugets_win: + name: Tests Windows (Requires Nugets) uses: ./.github/workflows/run-tests.yml - needs: [setup_for_tests_win, build_packages] - if: ${{ fromJson(needs.setup_for_tests_win.outputs.split_tests_matrix).include[0] != null }} + needs: [setup_for_tests, build_packages] + if: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets).include[0] != null }} strategy: fail-fast: false - matrix: ${{ fromJson(needs.setup_for_tests_win.outputs.split_tests_matrix) }} + matrix: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets) }} with: - testShortName: ${{ matrix.projectName }}_${{ matrix.shortname }} + testShortName: ${{ matrix.shortname }} os: "windows-latest" testProjectPath: ${{ matrix.testProjectPath }} testSessionTimeout: ${{ matrix.testSessionTimeout }} testHangTimeout: ${{ matrix.testHangTimeout }} - extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true ${{ matrix.filterArg }}" + extraTestArgs: "${{ matrix.extraTestArgs }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} requiresNugets: ${{ matrix.requiresNugets }} requiresTestSdk: ${{ matrix.requiresTestSdk }} @@ -268,15 +194,12 @@ jobs: needs: [ endtoend_tests, extension_tests_win, - integrations_test_lin, - integrations_test_macos, - integrations_test_win, - templates_test_lin, - templates_test_macos, - templates_test_win, - split_tests_lin, - split_tests_macos, - split_tests_win + tests_no_nugets_lin, + tests_no_nugets_macos, + tests_no_nugets_win, + tests_requires_nugets_lin, + tests_requires_nugets_macos, + tests_requires_nugets_win ] steps: - name: Checkout code From 59bb13d3a667c7cf9b911fcf245875d55eb4dcae Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 00:58:45 -0400 Subject: [PATCH 14/48] fixy --- .github/actions/enumerate-tests/action.yml | 2 +- .github/workflows/tests.yml | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/actions/enumerate-tests/action.yml b/.github/actions/enumerate-tests/action.yml index fcdc6f4f69f..cf619ccb762 100644 --- a/.github/actions/enumerate-tests/action.yml +++ b/.github/actions/enumerate-tests/action.yml @@ -56,7 +56,7 @@ runs: $testsNotRequiringNugets = @() foreach ($test in $matrixContent.include) { - if ($test.requiresNugets -eq $true) { + if ($test.requiresNugets -eq "true" -or $test.requiresNugets -eq $true) { $testsRequiringNugets += $test } else { $testsNotRequiringNugets += $test diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3dfeceb62f1..3b7ded1474c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -44,7 +44,7 @@ jobs: testProjectPath: ${{ matrix.testProjectPath }} testSessionTimeout: ${{ matrix.testSessionTimeout }} testHangTimeout: ${{ matrix.testHangTimeout }} - extraTestArgs: "${{ matrix.extraTestArgs }}" + extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.filterArg }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} requiresNugets: ${{ matrix.requiresNugets }} requiresTestSdk: ${{ matrix.requiresTestSdk }} @@ -64,7 +64,7 @@ jobs: testProjectPath: ${{ matrix.testProjectPath }} testSessionTimeout: ${{ matrix.testSessionTimeout }} testHangTimeout: ${{ matrix.testHangTimeout }} - extraTestArgs: "${{ matrix.extraTestArgs }}" + extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.filterArg }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} requiresNugets: ${{ matrix.requiresNugets }} requiresTestSdk: ${{ matrix.requiresTestSdk }} @@ -84,7 +84,7 @@ jobs: testProjectPath: ${{ matrix.testProjectPath }} testSessionTimeout: ${{ matrix.testSessionTimeout }} testHangTimeout: ${{ matrix.testHangTimeout }} - extraTestArgs: "${{ matrix.extraTestArgs }}" + extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.filterArg }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} requiresNugets: ${{ matrix.requiresNugets }} requiresTestSdk: ${{ matrix.requiresTestSdk }} @@ -104,7 +104,7 @@ jobs: testProjectPath: ${{ matrix.testProjectPath }} testSessionTimeout: ${{ matrix.testSessionTimeout }} testHangTimeout: ${{ matrix.testHangTimeout }} - extraTestArgs: "${{ matrix.extraTestArgs }}" + extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.filterArg }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} requiresNugets: ${{ matrix.requiresNugets }} requiresTestSdk: ${{ matrix.requiresTestSdk }} @@ -124,7 +124,7 @@ jobs: testProjectPath: ${{ matrix.testProjectPath }} testSessionTimeout: ${{ matrix.testSessionTimeout }} testHangTimeout: ${{ matrix.testHangTimeout }} - extraTestArgs: "${{ matrix.extraTestArgs }}" + extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.filterArg }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} requiresNugets: ${{ matrix.requiresNugets }} requiresTestSdk: ${{ matrix.requiresTestSdk }} @@ -144,7 +144,7 @@ jobs: testProjectPath: ${{ matrix.testProjectPath }} testSessionTimeout: ${{ matrix.testSessionTimeout }} testHangTimeout: ${{ matrix.testHangTimeout }} - extraTestArgs: "${{ matrix.extraTestArgs }}" + extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.filterArg }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} requiresNugets: ${{ matrix.requiresNugets }} requiresTestSdk: ${{ matrix.requiresTestSdk }} From 2bb57c35262fbed1c15ac9b4940351f6f24f5c19 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 01:01:08 -0400 Subject: [PATCH 15/48] improve naming --- eng/scripts/generate-test-matrix.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/scripts/generate-test-matrix.ps1 b/eng/scripts/generate-test-matrix.ps1 index 405dd8d83b7..d8706e45438 100644 --- a/eng/scripts/generate-test-matrix.ps1 +++ b/eng/scripts/generate-test-matrix.ps1 @@ -77,11 +77,12 @@ function Read-Metadata($file, $projectName) { } function New-EntryCollection($c,$meta) { + $projectShortName = $meta.projectName -replace '^Aspire\.' -replace '\.Tests$' [ordered]@{ type = 'collection' projectName = $meta.projectName name = $c - shortname = "Collection_$c" + shortname = "${projectShortName}_$c" testProjectPath = $meta.testProjectPath filterArg = "--filter-trait `"Partition=$c`"" requiresNugets = ($meta.requiresNugets -eq 'true') From dc0e26c2b4143963cbed62ac4fd2417e40b907fe Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 01:06:54 -0400 Subject: [PATCH 16/48] cleanup --- .github/workflows/tests.yml | 12 ++++++------ docs/test-splitting/STEP_06_CI_INTEGRATION.md | 12 ++++++------ eng/scripts/generate-test-matrix.ps1 | 10 +++++----- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3b7ded1474c..07357fb2ac9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -44,7 +44,7 @@ jobs: testProjectPath: ${{ matrix.testProjectPath }} testSessionTimeout: ${{ matrix.testSessionTimeout }} testHangTimeout: ${{ matrix.testHangTimeout }} - extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.filterArg }}" + extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} requiresNugets: ${{ matrix.requiresNugets }} requiresTestSdk: ${{ matrix.requiresTestSdk }} @@ -64,7 +64,7 @@ jobs: testProjectPath: ${{ matrix.testProjectPath }} testSessionTimeout: ${{ matrix.testSessionTimeout }} testHangTimeout: ${{ matrix.testHangTimeout }} - extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.filterArg }}" + extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} requiresNugets: ${{ matrix.requiresNugets }} requiresTestSdk: ${{ matrix.requiresTestSdk }} @@ -84,7 +84,7 @@ jobs: testProjectPath: ${{ matrix.testProjectPath }} testSessionTimeout: ${{ matrix.testSessionTimeout }} testHangTimeout: ${{ matrix.testHangTimeout }} - extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.filterArg }}" + extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} requiresNugets: ${{ matrix.requiresNugets }} requiresTestSdk: ${{ matrix.requiresTestSdk }} @@ -104,7 +104,7 @@ jobs: testProjectPath: ${{ matrix.testProjectPath }} testSessionTimeout: ${{ matrix.testSessionTimeout }} testHangTimeout: ${{ matrix.testHangTimeout }} - extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.filterArg }}" + extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} requiresNugets: ${{ matrix.requiresNugets }} requiresTestSdk: ${{ matrix.requiresTestSdk }} @@ -124,7 +124,7 @@ jobs: testProjectPath: ${{ matrix.testProjectPath }} testSessionTimeout: ${{ matrix.testSessionTimeout }} testHangTimeout: ${{ matrix.testHangTimeout }} - extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.filterArg }}" + extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} requiresNugets: ${{ matrix.requiresNugets }} requiresTestSdk: ${{ matrix.requiresTestSdk }} @@ -144,7 +144,7 @@ jobs: testProjectPath: ${{ matrix.testProjectPath }} testSessionTimeout: ${{ matrix.testSessionTimeout }} testHangTimeout: ${{ matrix.testHangTimeout }} - extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.filterArg }}" + extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} requiresNugets: ${{ matrix.requiresNugets }} requiresTestSdk: ${{ matrix.requiresTestSdk }} diff --git a/docs/test-splitting/STEP_06_CI_INTEGRATION.md b/docs/test-splitting/STEP_06_CI_INTEGRATION.md index 4e4664dfad7..3bbbc18466e 100644 --- a/docs/test-splitting/STEP_06_CI_INTEGRATION.md +++ b/docs/test-splitting/STEP_06_CI_INTEGRATION.md @@ -31,7 +31,7 @@ Uses matrix: fromJson(needs.setup.outputs.split_tests_matrix) For each matrix entry: - testShortName: ${{ matrix.shortname }} - testProjectPath: ${{ matrix.testProjectPath }} - - filterArg: ${{ matrix.filterArg }} + - extraTestArgs: ${{ matrix.extraTestArgs }} - requiresNugets: ${{ matrix.requiresNugets }} - etc. ``` @@ -43,9 +43,9 @@ The workflow expects these fields (all present in v3 output): ```yaml matrix: shortname: "Collection_DatabaseTests" # Used for job name - projectName: "Aspire.Hosting.Tests" # Used in filterArg + projectName: "Aspire.Hosting.Tests" # Used in extraTestArgs testProjectPath: "tests/..." # Which project to test - filterArg: "--filter-collection ..." # xUnit filter + extraTestArgs: "--filter-collection ..." # xUnit filter requiresNugets: true/false # Download packages? requiresTestSdk: true/false # Need test SDK? testSessionTimeout: "20m" # Timeout @@ -104,7 +104,7 @@ split_tests_lin / Aspire.Templates.Tests_StarterTemplateRunTests (ubuntu-latest) Each job: 1. Downloads built packages (if `requiresNugets: true`) 2. Installs test SDK (if `requiresTestSdk: true`) -3. Runs: `dotnet test ... -- ` +3. Runs: `dotnet test ... -- ` 4. Uploads test results **Expected Duration**: Varies by test, but should be significantly less than running all tests together @@ -195,7 +195,7 @@ Total: ~25 minutes (parallel) ``` - [ ] Verify filter arguments are correct: ```bash - jq '.include[] | {shortname, filterArg}' split-tests-matrix.json + jq '.include[] | {shortname, extraTestArgs}' split-tests-matrix.json ``` ## Common CI Issues @@ -227,7 +227,7 @@ Total: ~25 minutes (parallel) **Cause**: Filter argument didn't match any tests **Fix**: -1. Check `filterArg` in matrix JSON +1. Check `extraTestArgs` in matrix JSON 2. Verify collection/class names match actual test code 3. Check `TestClassNamesPrefix` matches namespace diff --git a/eng/scripts/generate-test-matrix.ps1 b/eng/scripts/generate-test-matrix.ps1 index d8706e45438..44e1760f6ad 100644 --- a/eng/scripts/generate-test-matrix.ps1 +++ b/eng/scripts/generate-test-matrix.ps1 @@ -18,7 +18,7 @@ name fullClassName (class mode only) testProjectPath - filterArg + extraTestArgs requiresNugets requiresTestSdk enablePlaywrightInstall @@ -84,7 +84,7 @@ function New-EntryCollection($c,$meta) { name = $c shortname = "${projectShortName}_$c" testProjectPath = $meta.testProjectPath - filterArg = "--filter-trait `"Partition=$c`"" + extraTestArgs = "--filter-trait `"Partition=$c`"" requiresNugets = ($meta.requiresNugets -eq 'true') requiresTestSdk = ($meta.requiresTestSdk -eq 'true') enablePlaywrightInstall = ($meta.enablePlaywrightInstall -eq 'true') @@ -104,7 +104,7 @@ function New-EntryUncollected($collections,$meta) { name = 'UncollectedTests' shortname = 'Uncollected' testProjectPath = $meta.testProjectPath - filterArg = ($filters -join ' ') + extraTestArgs = ($filters -join ' ') requiresNugets = ($meta.requiresNugets -eq 'true') requiresTestSdk = ($meta.requiresTestSdk -eq 'true') enablePlaywrightInstall = ($meta.enablePlaywrightInstall -eq 'true') @@ -126,7 +126,7 @@ function New-EntryClass($full,$meta) { shortname = $short fullClassName = $full testProjectPath = $meta.testProjectPath - filterArg = "--filter-class `"$full`"" + extraTestArgs = "--filter-class `"$full`"" requiresNugets = ($meta.requiresNugets -eq 'true') requiresTestSdk = ($meta.requiresTestSdk -eq 'true') enablePlaywrightInstall = ($meta.enablePlaywrightInstall -eq 'true') @@ -142,7 +142,7 @@ function New-EntryRegular($shortName) { name = $shortName shortname = $shortName testProjectPath = "tests/Aspire.$shortName.Tests/Aspire.$shortName.Tests.csproj" - filterArg = "" + extraTestArgs = "" requiresNugets = $false requiresTestSdk = $false enablePlaywrightInstall = $false From fbbf166f1d5e9364ef22a3e61dc442e7e10a7ecb Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 01:29:06 -0400 Subject: [PATCH 17/48] fixy --- .github/actions/enumerate-tests/action.yml | 1 + .github/workflows/tests.yml | 13 ----- .../TestEnumerationRunsheetBuilder.targets | 4 +- eng/Testing.targets | 47 +++++++++++++++++++ 4 files changed, 51 insertions(+), 14 deletions(-) diff --git a/.github/actions/enumerate-tests/action.yml b/.github/actions/enumerate-tests/action.yml index cf619ccb762..1836ebe80de 100644 --- a/.github/actions/enumerate-tests/action.yml +++ b/.github/actions/enumerate-tests/action.yml @@ -92,3 +92,4 @@ runs: path: | artifacts/log/**/*.binlog artifacts/**/*.list + artifacts/combined-test-matrix.json diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 07357fb2ac9..c8c22725b3e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -150,18 +150,6 @@ jobs: requiresTestSdk: ${{ matrix.requiresTestSdk }} enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} - endtoend_tests: - name: EndToEnd Linux - uses: ./.github/workflows/run-tests.yml - needs: build_packages - with: - testShortName: EndToEnd - # EndToEnd is not run on Windows/macOS due to missing Docker support - os: ubuntu-latest - testProjectPath: tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj - requiresNugets: true - versionOverrideArg: ${{ inputs.versionOverrideArg }} - extension_tests_win: name: Run VS Code extension tests (Windows) runs-on: windows-latest @@ -192,7 +180,6 @@ jobs: runs-on: ubuntu-latest name: Final Test Results needs: [ - endtoend_tests, extension_tests_win, tests_no_nugets_lin, tests_no_nugets_macos, diff --git a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets index 7778cf3ea63..662263d95f5 100644 --- a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets +++ b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets @@ -44,13 +44,15 @@ <_RelativeProjectPath>$([System.String]::Copy('$(MSBuildProjectFullPath)').Replace('$(RepoRoot)', '')) + + <_RelativeProjectPath Condition="$(_RelativeProjectPath.StartsWith('/')) or $(_RelativeProjectPath.StartsWith('\'))">$(_RelativeProjectPath.Substring(1)) <_HelixDir>$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix')) diff --git a/eng/Testing.targets b/eng/Testing.targets index 684b6d9fa25..28ff2f27401 100644 --- a/eng/Testing.targets +++ b/eng/Testing.targets @@ -146,4 +146,51 @@ + + + + <_HelixDir>$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix')) + <_MetadataFile>$(_HelixDir)$(MSBuildProjectName).tests.metadata.json + + + <_RelativeProjectPath>$([System.String]::Copy('$(MSBuildProjectFullPath)').Replace('$(RepoRoot)', '')) + + <_RelativeProjectPath Condition="$(_RelativeProjectPath.StartsWith('/')) or $(_RelativeProjectPath.StartsWith('\\'))">$(_RelativeProjectPath.Substring(1)) + + + <_RequiresNugets>false + <_RequiresNugets Condition="'$(RequiresNugetsForSplitTests)' == 'true'">true + + + <_RequiresTestSdk>false + <_RequiresTestSdk Condition="'$(RequiresTestSdkForSplitTests)' == 'true'">true + + + <_MetadataJson>{ + "projectName": "$(MSBuildProjectName)", + "testClassNamesPrefix": "$(MSBuildProjectName)", + "testProjectPath": "$(_RelativeProjectPath)", + "requiresNugets": "$(_RequiresNugets.ToLowerInvariant())", + "requiresTestSdk": "$(_RequiresTestSdk.ToLowerInvariant())", + "enablePlaywrightInstall": "false", + "testSessionTimeout": "20m", + "testHangTimeout": "10m", + "uncollectedTestsSessionTimeout": "15m", + "uncollectedTestsHangTimeout": "10m" +} + + + + + + + + + From ec76d624e2fb0f3c82d5fd0cdb2c00df057e8b69 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 01:35:07 -0400 Subject: [PATCH 18/48] more partitions --- tests/Aspire.Hosting.Tests/MSBuildTests.cs | 1 + .../Publishing/ResourceContainerImageBuilderTests.cs | 7 ++++--- tests/Aspire.Hosting.Tests/WithUrlsTests.cs | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/Aspire.Hosting.Tests/MSBuildTests.cs b/tests/Aspire.Hosting.Tests/MSBuildTests.cs index c1501234b68..d2072cb5506 100644 --- a/tests/Aspire.Hosting.Tests/MSBuildTests.cs +++ b/tests/Aspire.Hosting.Tests/MSBuildTests.cs @@ -6,6 +6,7 @@ namespace Aspire.Hosting.Tests; +[Trait("Partition", "Partition1")] public class MSBuildTests { /// diff --git a/tests/Aspire.Hosting.Tests/Publishing/ResourceContainerImageBuilderTests.cs b/tests/Aspire.Hosting.Tests/Publishing/ResourceContainerImageBuilderTests.cs index 7af0a8d3306..48e9a7c722f 100644 --- a/tests/Aspire.Hosting.Tests/Publishing/ResourceContainerImageBuilderTests.cs +++ b/tests/Aspire.Hosting.Tests/Publishing/ResourceContainerImageBuilderTests.cs @@ -13,6 +13,7 @@ namespace Aspire.Hosting.Tests.Publishing; +[Trait("Partition", "Partition1")] public class ResourceContainerImageBuilderTests(ITestOutputHelper output) { [Fact] @@ -803,15 +804,15 @@ public async Task ResolveValue_FormatsDecimalWithInvariantCulture() // Test decimal value var result = await ResourceContainerImageBuilder.ResolveValue(3.14, CancellationToken.None); Assert.Equal("3.14", result); - + // Test double value result = await ResourceContainerImageBuilder.ResolveValue(3.14d, CancellationToken.None); Assert.Equal("3.14", result); - + // Test float value result = await ResourceContainerImageBuilder.ResolveValue(3.14f, CancellationToken.None); Assert.Equal("3.14", result); - + // Test integer (should also work) result = await ResourceContainerImageBuilder.ResolveValue(42, CancellationToken.None); Assert.Equal("42", result); diff --git a/tests/Aspire.Hosting.Tests/WithUrlsTests.cs b/tests/Aspire.Hosting.Tests/WithUrlsTests.cs index ced837ea628..8c2fe7a2376 100644 --- a/tests/Aspire.Hosting.Tests/WithUrlsTests.cs +++ b/tests/Aspire.Hosting.Tests/WithUrlsTests.cs @@ -11,6 +11,7 @@ namespace Aspire.Hosting.Tests; +[Trait("Partition", "Partition1")] public class WithUrlsTests { [Fact] From cee0dde9a015f2b37bcac137920e8017f3cc709b Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 01:39:55 -0400 Subject: [PATCH 19/48] fixy --- eng/scripts/generate-test-matrix.ps1 | 35 +++++++++++++++++++++--- eng/scripts/process-test-enumeration.ps1 | 11 ++++++-- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/eng/scripts/generate-test-matrix.ps1 b/eng/scripts/generate-test-matrix.ps1 index 44e1760f6ad..b9d44c28056 100644 --- a/eng/scripts/generate-test-matrix.ps1 +++ b/eng/scripts/generate-test-matrix.ps1 @@ -201,10 +201,37 @@ foreach ($lf in $listFiles) { # Add regular (non-split) test projects if provided if ($RegularTestProjectsFile -and (Test-Path $RegularTestProjectsFile)) { - $regularProjects = @(Get-Content $RegularTestProjectsFile | Where-Object { $_ -and -not [string]::IsNullOrWhiteSpace($_) }) - Write-Host "Adding $($regularProjects.Count) regular test project(s)" - foreach ($shortName in $regularProjects) { - $entries.Add( (New-EntryRegular $shortName) ) | Out-Null + # Check if JSON file exists with full metadata + $jsonFile = "$RegularTestProjectsFile.json" + if (Test-Path $jsonFile) { + $regularProjectsData = Get-Content -Raw $jsonFile | ConvertFrom-Json + if ($regularProjectsData -isnot [Array]) { + $regularProjectsData = @($regularProjectsData) + } + Write-Host "Adding $($regularProjectsData.Count) regular test project(s) from JSON" + foreach ($proj in $regularProjectsData) { + $entry = [ordered]@{ + type = 'regular' + projectName = $proj.project + name = $proj.shortName + shortname = $proj.shortName + testProjectPath = $proj.fullPath + extraTestArgs = "" + requiresNugets = $false + requiresTestSdk = $false + enablePlaywrightInstall = $false + testSessionTimeout = '20m' + testHangTimeout = '10m' + } + $entries.Add($entry) | Out-Null + } + } else { + # Fallback to old behavior for backward compatibility + $regularProjects = @(Get-Content $RegularTestProjectsFile | Where-Object { $_ -and -not [string]::IsNullOrWhiteSpace($_) }) + Write-Host "Adding $($regularProjects.Count) regular test project(s) (legacy mode)" + foreach ($shortName in $regularProjects) { + $entries.Add( (New-EntryRegular $shortName) ) | Out-Null + } } } diff --git a/eng/scripts/process-test-enumeration.ps1 b/eng/scripts/process-test-enumeration.ps1 index ef1830406b8..3d091db44ef 100644 --- a/eng/scripts/process-test-enumeration.ps1 +++ b/eng/scripts/process-test-enumeration.ps1 @@ -50,7 +50,8 @@ foreach ($file in $enumerationFiles) { if ($content.splitTests -eq 'true') { $splitTestProjects += $content.shortName } else { - $regularTestProjects += $content.shortName + # Store full enumeration data for regular tests + $regularTestProjects += $content } Write-Host " Included: $($content.shortName) (Split: $($content.splitTests))" } else { @@ -71,8 +72,12 @@ if (-not (Test-Path $outputDir)) { New-Item -Path $outputDir -ItemType Directory -Force | Out-Null } -# Write regular test projects list -$regularTestProjects | Set-Content $TestsListOutputPath +# Write regular test projects list as JSON for matrix generation +if ($regularTestProjects.Count -gt 0) { + $regularTestProjects | ConvertTo-Json -Depth 10 | Set-Content "$TestsListOutputPath.json" +} +# Also write just the short names for backward compatibility +$regularTestProjects | ForEach-Object { $_.shortName } | Set-Content $TestsListOutputPath # Write split test projects list if any exist if ($splitTestProjects.Count -gt 0) { From 1b4dc14cbb2239c04c41ec7a67685adfd080ae2e Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 01:44:22 -0400 Subject: [PATCH 20/48] fixy-more --- eng/Testing.targets | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/eng/Testing.targets b/eng/Testing.targets index 28ff2f27401..80748b630b0 100644 --- a/eng/Testing.targets +++ b/eng/Testing.targets @@ -169,6 +169,16 @@ <_RequiresTestSdk>false <_RequiresTestSdk Condition="'$(RequiresTestSdkForSplitTests)' == 'true'">true + + <_EnablePlaywrightInstall>false + <_EnablePlaywrightInstall Condition="'$(EnablePlaywrightInstallForSplitTests)' == 'true'">true + + + <_TestSessionTimeout Condition="'$(SplitTestSessionTimeout)' != ''">$(SplitTestSessionTimeout) + <_TestSessionTimeout Condition="'$(SplitTestSessionTimeout)' == ''">20m + <_TestHangTimeout Condition="'$(SplitTestHangTimeout)' != ''">$(SplitTestHangTimeout) + <_TestHangTimeout Condition="'$(SplitTestHangTimeout)' == ''">10m + <_MetadataJson>{ "projectName": "$(MSBuildProjectName)", @@ -176,9 +186,9 @@ "testProjectPath": "$(_RelativeProjectPath)", "requiresNugets": "$(_RequiresNugets.ToLowerInvariant())", "requiresTestSdk": "$(_RequiresTestSdk.ToLowerInvariant())", - "enablePlaywrightInstall": "false", - "testSessionTimeout": "20m", - "testHangTimeout": "10m", + "enablePlaywrightInstall": "$(_EnablePlaywrightInstall.ToLowerInvariant())", + "testSessionTimeout": "$(_TestSessionTimeout)", + "testHangTimeout": "$(_TestHangTimeout)", "uncollectedTestsSessionTimeout": "15m", "uncollectedTestsHangTimeout": "10m" } @@ -190,7 +200,7 @@ Lines="$(_MetadataJson)" Overwrite="true" /> - + From c4c18ea4d8a825bc77d0c7db2a9c1fbb51e83f26 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 01:52:38 -0400 Subject: [PATCH 21/48] fixy-more --- .../TestEnumerationRunsheetBuilder.targets | 4 ++-- eng/Testing.targets | 12 ++++++------ tests/Aspire.Hosting.Tests/SlimTestProgramTests.cs | 1 - 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets index 662263d95f5..24cfd4df08f 100644 --- a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets +++ b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets @@ -22,7 +22,7 @@ <_ShouldSkipProject Condition="$(MSBuildProjectDirectory.Contains('tests\Shared'))">true <_ShouldSkipProject Condition="$(MSBuildProjectDirectory.Contains('tests\testproject'))">true <_ShouldSkipProject Condition="$(MSBuildProjectDirectory.Contains('tests\TestingAppHost1'))">true - <_ShouldSkipProject Condition="'$(MSBuildProjectName)' == 'Aspire.EndToEnd.Tests'">true + <_ShortName>$([System.IO.Path]::GetFileNameWithoutExtension('$(MSBuildProjectName)').Replace('Aspire.', '').Replace('.Tests', '')) @@ -93,4 +93,4 @@ Condition="'$(_ShouldSkipProject)' == 'true'" /> - \ No newline at end of file + diff --git a/eng/Testing.targets b/eng/Testing.targets index 80748b630b0..ee5c223b23b 100644 --- a/eng/Testing.targets +++ b/eng/Testing.targets @@ -155,30 +155,30 @@ <_HelixDir>$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix')) <_MetadataFile>$(_HelixDir)$(MSBuildProjectName).tests.metadata.json - + <_RelativeProjectPath>$([System.String]::Copy('$(MSBuildProjectFullPath)').Replace('$(RepoRoot)', '')) <_RelativeProjectPath Condition="$(_RelativeProjectPath.StartsWith('/')) or $(_RelativeProjectPath.StartsWith('\\'))">$(_RelativeProjectPath.Substring(1)) - + <_RequiresNugets>false <_RequiresNugets Condition="'$(RequiresNugetsForSplitTests)' == 'true'">true - + <_RequiresTestSdk>false <_RequiresTestSdk Condition="'$(RequiresTestSdkForSplitTests)' == 'true'">true - + <_EnablePlaywrightInstall>false <_EnablePlaywrightInstall Condition="'$(EnablePlaywrightInstallForSplitTests)' == 'true'">true - + <_TestSessionTimeout Condition="'$(SplitTestSessionTimeout)' != ''">$(SplitTestSessionTimeout) <_TestSessionTimeout Condition="'$(SplitTestSessionTimeout)' == ''">20m <_TestHangTimeout Condition="'$(SplitTestHangTimeout)' != ''">$(SplitTestHangTimeout) <_TestHangTimeout Condition="'$(SplitTestHangTimeout)' == ''">10m - + <_MetadataJson>{ "projectName": "$(MSBuildProjectName)", diff --git a/tests/Aspire.Hosting.Tests/SlimTestProgramTests.cs b/tests/Aspire.Hosting.Tests/SlimTestProgramTests.cs index 2348849c256..ab3dd81ccc1 100644 --- a/tests/Aspire.Hosting.Tests/SlimTestProgramTests.cs +++ b/tests/Aspire.Hosting.Tests/SlimTestProgramTests.cs @@ -8,7 +8,6 @@ namespace Aspire.Hosting.Tests; [Collection("SlimTestProgram")] -[Trait("Partition", "SlimTestProgram")] public class SlimTestProgramTests { private readonly SlimTestProgramFixture _slimTestProgramFixture; From b090b4ebb65f53feb94bc3b27902d2829c840859 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 02:01:38 -0400 Subject: [PATCH 22/48] fix-json --- .github/actions/enumerate-tests/action.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/enumerate-tests/action.yml b/.github/actions/enumerate-tests/action.yml index 1836ebe80de..8b20e4d43f2 100644 --- a/.github/actions/enumerate-tests/action.yml +++ b/.github/actions/enumerate-tests/action.yml @@ -70,8 +70,8 @@ runs: $nugetMatrixJson = ConvertTo-Json $nugetMatrix -Compress -Depth 10 $nonNugetMatrixJson = ConvertTo-Json $nonNugetMatrix -Compress -Depth 10 - "tests_matrix_requires_nugets=$nugetMatrixJson" | Out-File -FilePath $env:GITHUB_OUTPUT - "tests_matrix_no_nugets=$nonNugetMatrixJson" | Out-File -FilePath $env:GITHUB_OUTPUT + "tests_matrix_requires_nugets=$nugetMatrixJson" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + "tests_matrix_no_nugets=$nonNugetMatrixJson" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append Write-Host "Combined test matrices generated successfully" Write-Host "Tests requiring nugets: $($testsRequiringNugets.Count)" @@ -79,8 +79,8 @@ runs: } else { # Empty matrices if no combined matrix found $emptyMatrix = '{"include":[]}' - "tests_matrix_requires_nugets=$emptyMatrix" | Out-File -FilePath $env:GITHUB_OUTPUT - "tests_matrix_no_nugets=$emptyMatrix" | Out-File -FilePath $env:GITHUB_OUTPUT + "tests_matrix_requires_nugets=$emptyMatrix" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + "tests_matrix_no_nugets=$emptyMatrix" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append Write-Host "No combined test matrix found, using empty matrices" } From f65c923d829fe643819f2042c8460ae031bd869d Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 02:15:37 -0400 Subject: [PATCH 23/48] cleanup --- eng/scripts/extract-test-metadata.ps1 | 42 +++++++++---------- .../Aspire.EndToEnd.Tests.csproj | 2 + tools/ExtractTestPartitions/Program.cs | 39 +++++++---------- 3 files changed, 38 insertions(+), 45 deletions(-) diff --git a/eng/scripts/extract-test-metadata.ps1 b/eng/scripts/extract-test-metadata.ps1 index ae507f1bc14..ffa2ba3b7ae 100644 --- a/eng/scripts/extract-test-metadata.ps1 +++ b/eng/scripts/extract-test-metadata.ps1 @@ -83,40 +83,40 @@ $classes = [System.Collections.Generic.HashSet[string]]::new() $partitionsFile = [System.IO.Path]::GetTempFileName() try { $toolPath = Join-Path $RepoRoot "artifacts/bin/ExtractTestPartitions/Debug/net8.0/ExtractTestPartitions.dll" - + # Build the tool if it doesn't exist if (-not (Test-Path $toolPath)) { Write-Host "Building ExtractTestPartitions tool..." $toolProjectPath = Join-Path $RepoRoot "tools/ExtractTestPartitions/ExtractTestPartitions.csproj" & dotnet build $toolProjectPath -c Debug --nologo -v quiet if ($LASTEXITCODE -ne 0) { - Write-Warning "Failed to build ExtractTestPartitions tool. Falling back to class-based mode." + Write-Error "Failed to build ExtractTestPartitions tool." } } - # Run the tool if available - if (Test-Path $toolPath) { - Write-Host "Extracting partitions from assembly: $TestAssemblyPath" - & dotnet $toolPath --assembly-path $TestAssemblyPath --output-file $partitionsFile 2>&1 | Write-Host - - if ($LASTEXITCODE -eq 0 -and (Test-Path $partitionsFile)) { - $partitionLines = Get-Content $partitionsFile -ErrorAction SilentlyContinue - if ($partitionLines) { - foreach ($partition in $partitionLines) { - if (-not [string]::IsNullOrWhiteSpace($partition)) { - $collections.Add($partition.Trim()) | Out-Null - } - } - Write-Host "Found $($collections.Count) partition(s) via attribute extraction" + Write-Host "Extracting partitions from assembly: $TestAssemblyPath" + & dotnet $toolPath --assembly-path $TestAssemblyPath --output-file $partitionsFile 2>&1 | Write-Host + # throw on failure + if ($LASTEXITCODE -ne 0) { + throw "Failed to extract partitions from assembly." + } + + # throw if partitions file missing + if (-not (Test-Path $partitionsFile)) { + throw "Partitions file not created by ExtractTestPartitions tool." + } + + $partitionLines = Get-Content $partitionsFile -ErrorAction SilentlyContinue + if ($partitionLines) { + foreach ($partition in $partitionLines) { + if (-not [string]::IsNullOrWhiteSpace($partition)) { + $collections.Add($partition.Trim()) | Out-Null } } + Write-Host "Found $($collections.Count) partition(s) via attribute extraction" } } catch { Write-Warning "Error running ExtractTestPartitions tool: $_" -} finally { - if (Test-Path $partitionsFile) { - Remove-Item $partitionsFile -Force - } } # Extract class names from test listing @@ -181,4 +181,4 @@ if ($MetadataJsonFile -and (Test-Path $MetadataJsonFile)) { Write-Host "Mode: $mode" Write-Host "Collections discovered (after filtering): $($filteredCollections.Count)" Write-Host "Classes discovered: $($classes.Count)" -Write-Host "Output list written: $OutputListFile" \ No newline at end of file +Write-Host "Output list written: $OutputListFile" diff --git a/tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj b/tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj index ae826d9bb89..3ac5f69bbd1 100644 --- a/tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj +++ b/tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj @@ -17,6 +17,8 @@ false testassets\testproject\ + true + <_BuildForTestsRunningOutsideOfRepo Condition="'$(TestsRunningOutsideOfRepo)' == 'true' or '$(ContinuousIntegrationBuild)' == 'true'">true $(_BuildForTestsRunningOutsideOfRepo) BUILD_FOR_TESTS_RUNNING_OUTSIDE_OF_REPO;$(DefineConstants) diff --git a/tools/ExtractTestPartitions/Program.cs b/tools/ExtractTestPartitions/Program.cs index 8449a0d45ab..ab44615c8fd 100644 --- a/tools/ExtractTestPartitions/Program.cs +++ b/tools/ExtractTestPartitions/Program.cs @@ -67,33 +67,24 @@ static void ExtractPartitions(string assemblyPath, string outputFile) { var attrTypeName = attr.AttributeType.FullName ?? attr.AttributeType.Name; - // Check for [Collection("name")] attribute - if (attrTypeName.EndsWith(".CollectionAttribute") || attrTypeName == "CollectionAttribute") + if (!attrTypeName.EndsWith(".TraitAttribute") && attrTypeName != "TraitAttribute") { - if (attr.ConstructorArguments.Count > 0 && attr.ConstructorArguments[0].Value is string collectionName) - { - if (!string.IsNullOrWhiteSpace(collectionName)) - { - partitions.Add(collectionName); - Console.WriteLine($"Found Collection: {collectionName} on {type.Name}"); - } - } + continue; } - // Check for [Trait("Partition", "name")] attribute - else if (attrTypeName.EndsWith(".TraitAttribute") || attrTypeName == "TraitAttribute") + + if (attr.ConstructorArguments.Count < 2) + { + continue; + } + + var key = attr.ConstructorArguments[0].Value as string; + var value = attr.ConstructorArguments[1].Value as string; + + if (key?.Equals("Partition", StringComparison.OrdinalIgnoreCase) == true && + !string.IsNullOrWhiteSpace(value)) { - if (attr.ConstructorArguments.Count >= 2) - { - var key = attr.ConstructorArguments[0].Value as string; - var value = attr.ConstructorArguments[1].Value as string; - - if (key?.Equals("Partition", StringComparison.OrdinalIgnoreCase) == true && - !string.IsNullOrWhiteSpace(value)) - { - partitions.Add(value); - Console.WriteLine($"Found Trait Partition: {value} on {type.Name}"); - } - } + partitions.Add(value); + Console.WriteLine($"Found Trait Partition: {value} on {type.Name}"); } } } From 183e9151a1f3fe0a8b9a2c9eb6569f25bf1a5fe7 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 02:27:46 -0400 Subject: [PATCH 24/48] cleanup --- eng/scripts/extract-test-metadata.ps1 | 52 +++++++++++++++++---------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/eng/scripts/extract-test-metadata.ps1 b/eng/scripts/extract-test-metadata.ps1 index ffa2ba3b7ae..eb9598fece9 100644 --- a/eng/scripts/extract-test-metadata.ps1 +++ b/eng/scripts/extract-test-metadata.ps1 @@ -80,7 +80,8 @@ $collections = [System.Collections.Generic.HashSet[string]]::new() $classes = [System.Collections.Generic.HashSet[string]]::new() # Extract partitions using the ExtractTestPartitions tool -$partitionsFile = [System.IO.Path]::GetTempFileName() +# This step is optional - if it fails, we'll fall back to class-based splitting +$partitionsFile = Join-Path ([System.IO.Path]::GetTempPath()) "partitions-$([System.Guid]::NewGuid()).txt" try { $toolPath = Join-Path $RepoRoot "artifacts/bin/ExtractTestPartitions/Debug/net8.0/ExtractTestPartitions.dll" @@ -90,33 +91,46 @@ try { $toolProjectPath = Join-Path $RepoRoot "tools/ExtractTestPartitions/ExtractTestPartitions.csproj" & dotnet build $toolProjectPath -c Debug --nologo -v quiet if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to build ExtractTestPartitions tool." + Write-Host "Warning: Failed to build ExtractTestPartitions tool. Using class-based splitting." } } - Write-Host "Extracting partitions from assembly: $TestAssemblyPath" - & dotnet $toolPath --assembly-path $TestAssemblyPath --output-file $partitionsFile 2>&1 | Write-Host - # throw on failure - if ($LASTEXITCODE -ne 0) { - throw "Failed to extract partitions from assembly." - } + if (Test-Path $toolPath) { + Write-Host "Extracting partitions from assembly: $TestAssemblyPath" + $toolOutput = & dotnet $toolPath --assembly-path $TestAssemblyPath --output-file $partitionsFile 2>&1 + $toolExitCode = $LASTEXITCODE - # throw if partitions file missing - if (-not (Test-Path $partitionsFile)) { - throw "Partitions file not created by ExtractTestPartitions tool." - } + # Display tool output (informational) + if ($toolOutput) { + $toolOutput | Write-Host + } - $partitionLines = Get-Content $partitionsFile -ErrorAction SilentlyContinue - if ($partitionLines) { - foreach ($partition in $partitionLines) { - if (-not [string]::IsNullOrWhiteSpace($partition)) { - $collections.Add($partition.Trim()) | Out-Null + # If partitions file was created, read it (even if exit code is non-zero) + if (Test-Path $partitionsFile) { + $partitionLines = Get-Content $partitionsFile -ErrorAction SilentlyContinue + if ($partitionLines) { + foreach ($partition in $partitionLines) { + if (-not [string]::IsNullOrWhiteSpace($partition)) { + $collections.Add($partition.Trim()) | Out-Null + } + } + Write-Host "Found $($collections.Count) partition(s) via attribute extraction" } } - Write-Host "Found $($collections.Count) partition(s) via attribute extraction" + elseif ($toolExitCode -ne 0) { + Write-Host "Partition extraction completed with warnings. Falling back to class-based splitting." + } } } catch { - Write-Warning "Error running ExtractTestPartitions tool: $_" + # Partition extraction is optional - if it fails, we fall back to class-based splitting + Write-Host "Partition extraction encountered an issue. Falling back to class-based splitting." + Write-Host "Details: $_" +} +finally { + # Clean up temp file + if (Test-Path $partitionsFile) { + Remove-Item $partitionsFile -ErrorAction SilentlyContinue + } } # Extract class names from test listing From 5a8472c61c92a0cce8825e5eafee8809d90d3edc Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 02:31:25 -0400 Subject: [PATCH 25/48] fix-e2e --- .../TestEnumerationRunsheetBuilder.targets | 8 ++--- eng/scripts/generate-test-matrix.ps1 | 35 ++++++++++++++++--- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets index 24cfd4df08f..a8c1284804f 100644 --- a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets +++ b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets @@ -41,11 +41,11 @@ - + + Condition="'$(_ShouldSkipProject)' != 'true' and '@(_ProjectInfo->Count())' > 0" /> @@ -54,11 +54,11 @@ <_RelativeProjectPath Condition="$(_RelativeProjectPath.StartsWith('/')) or $(_RelativeProjectPath.StartsWith('\'))">$(_RelativeProjectPath.Substring(1)) - + <_HelixDir>$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix')) <_TestListFile>$(_HelixDir)$(MSBuildProjectName).tests.list <_MetadataFile>$(_HelixDir)$(MSBuildProjectName).tests.metadata.json - <_HasTestMetadata Condition="'%(_ProjectInfo.SplitTests)' == 'true' and Exists('$(_TestListFile)') and Exists('$(_MetadataFile)')">true + <_HasTestMetadata Condition="Exists('$(_MetadataFile)')">true <_HasTestMetadata Condition="'$(_HasTestMetadata)' != 'true'">false diff --git a/eng/scripts/generate-test-matrix.ps1 b/eng/scripts/generate-test-matrix.ps1 index b9d44c28056..1cf2b605805 100644 --- a/eng/scripts/generate-test-matrix.ps1 +++ b/eng/scripts/generate-test-matrix.ps1 @@ -210,6 +210,31 @@ if ($RegularTestProjectsFile -and (Test-Path $RegularTestProjectsFile)) { } Write-Host "Adding $($regularProjectsData.Count) regular test project(s) from JSON" foreach ($proj in $regularProjectsData) { + # Try to read metadata file for this project if it exists + $metadataFile = $null + if ($proj.metadataFile) { + # metadataFile path is relative to repo root, so make it absolute + $metadataFile = Join-Path $TestListsDirectory ".." ($proj.metadataFile -replace '^artifacts/', '') + } + + $meta = $null + if ($metadataFile -and (Test-Path $metadataFile)) { + $meta = Read-Metadata $metadataFile $proj.project + Write-Host " Loaded metadata for $($proj.project) from $metadataFile (requiresNugets=$($meta.requiresNugets))" + } else { + # Use defaults if no metadata file exists + $meta = @{ + projectName = $proj.project + testProjectPath = $proj.fullPath + requiresNugets = 'false' + requiresTestSdk = 'false' + enablePlaywrightInstall = 'false' + testSessionTimeout = '20m' + testHangTimeout = '10m' + } + Write-Host " Using default metadata for $($proj.project) (no metadata file found)" + } + $entry = [ordered]@{ type = 'regular' projectName = $proj.project @@ -217,11 +242,11 @@ if ($RegularTestProjectsFile -and (Test-Path $RegularTestProjectsFile)) { shortname = $proj.shortName testProjectPath = $proj.fullPath extraTestArgs = "" - requiresNugets = $false - requiresTestSdk = $false - enablePlaywrightInstall = $false - testSessionTimeout = '20m' - testHangTimeout = '10m' + requiresNugets = ($meta.requiresNugets -eq 'true') + requiresTestSdk = ($meta.requiresTestSdk -eq 'true') + enablePlaywrightInstall = ($meta.enablePlaywrightInstall -eq 'true') + testSessionTimeout = $meta.testSessionTimeout + testHangTimeout = $meta.testHangTimeout } $entries.Add($entry) | Out-Null } From f10bfed2a9a3251b7714abe21792647bb047b52b Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 02:48:49 -0400 Subject: [PATCH 26/48] fixy --- eng/Testing.targets | 4 +- eng/scripts/generate-test-matrix.ps1 | 2 +- tests/Shared/GetTestProjects.proj | 134 --------------------------- 3 files changed, 3 insertions(+), 137 deletions(-) delete mode 100644 tests/Shared/GetTestProjects.proj diff --git a/eng/Testing.targets b/eng/Testing.targets index ee5c223b23b..a4c0e2f4a55 100644 --- a/eng/Testing.targets +++ b/eng/Testing.targets @@ -147,11 +147,11 @@ + Condition="'$(PrepareForHelix)' == 'true' and '$(IsTestProject)' == 'true'"> <_HelixDir>$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix')) <_MetadataFile>$(_HelixDir)$(MSBuildProjectName).tests.metadata.json diff --git a/eng/scripts/generate-test-matrix.ps1 b/eng/scripts/generate-test-matrix.ps1 index 1cf2b605805..13d89f82ab2 100644 --- a/eng/scripts/generate-test-matrix.ps1 +++ b/eng/scripts/generate-test-matrix.ps1 @@ -232,7 +232,7 @@ if ($RegularTestProjectsFile -and (Test-Path $RegularTestProjectsFile)) { testSessionTimeout = '20m' testHangTimeout = '10m' } - Write-Host " Using default metadata for $($proj.project) (no metadata file found)" + Write-Host " Using default metadata for $($proj.project) (no metadata file found at $metadataFile)" } $entry = [ordered]@{ diff --git a/tests/Shared/GetTestProjects.proj b/tests/Shared/GetTestProjects.proj deleted file mode 100644 index 7370dea356c..00000000000 --- a/tests/Shared/GetTestProjects.proj +++ /dev/null @@ -1,134 +0,0 @@ - - - - - $([MSBuild]::NormalizeDirectory($(MSBuildThisFileDirectory), '..', '..')) - $([MSBuild]::NormalizeDirectory($(RepoRoot), 'artifacts')) - - - - - - - <_BuildOs Condition="'$(BuildOs)' == ''">linux - <_BuildOs Condition="'$(BuildOs)' != ''">$(BuildOs) - - - - <_TestProjectsToExclude Include="$(RepoRoot)tests\Shared\**\*Tests.csproj" /> - <_TestProjectsToExclude Include="$(RepoRoot)tests\testproject\**\*Tests.csproj" /> - <_TestProjectsToExclude Include="$(RepoRoot)tests\TestingAppHost1\**\*Tests.csproj" /> - - - <_TestProjectsToExclude Include="$(RepoRoot)tests\Aspire.EndToEnd.Tests\**\*Tests.csproj" /> - - <_TestProjects Include="$(RepoRoot)tests\**\*Tests.csproj" - Exclude="@(_TestProjectsToExclude)" /> - - - - - - - - - - - - - - - - - - - - - - - - - - - <_BuildOs Condition="'$(BuildOs)' == ''">linux - <_BuildOs Condition="'$(BuildOs)' != ''">$(BuildOs) - - - - <_TestProjectsToExclude Include="$(RepoRoot)tests\Shared\**\*Tests.csproj" /> - <_TestProjectsToExclude Include="$(RepoRoot)tests\testproject\**\*Tests.csproj" /> - <_TestProjectsToExclude Include="$(RepoRoot)tests\TestingAppHost1\**\*Tests.csproj" /> - - - <_TestProjectsToExclude Include="$(RepoRoot)tests\Aspire.EndToEnd.Tests\**\*Tests.csproj" /> - - <_TestProjects Include="$(RepoRoot)tests\**\*Tests.csproj" - Exclude="@(_TestProjectsToExclude)" /> - - - - - - - - - <_ProjectsForSplitCheck Remove="@(_ProjectsForSplitCheck)" Condition="'%(RunTestsOnGithubActions)' != 'true' or '%(SplitTests)' != 'true'" /> - - - - - - - - - - - <_TestListsDir>$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix')) - <_BuildOs Condition="'$(BuildOs)' == ''">unknown - <_BuildOs Condition="'$(BuildOs)' != ''">$(BuildOs) - - - - - - - - - <_PwshCommand>pwsh - <_PwshCommand Condition="'$(OS)' == 'Windows_NT'">powershell - <_MatrixScriptPath>$([MSBuild]::NormalizePath($(RepoRoot), 'eng', 'scripts', 'generate-test-matrix.ps1')) - <_RegularTestProjectsFile>$([MSBuild]::NormalizePath($(MSBuildProjectDirectory), $(TestsListOutputPath))) - <_MatrixCommand>$(_PwshCommand) -NoProfile -ExecutionPolicy Bypass -File "$(_MatrixScriptPath)" - <_MatrixCommand>$(_MatrixCommand) -TestListsDirectory "$(_TestListsDir)" - <_MatrixCommand>$(_MatrixCommand) -OutputDirectory "$(TestMatrixOutputPath)" - <_MatrixCommand>$(_MatrixCommand) -BuildOs "$(_BuildOs)" - <_MatrixCommand>$(_MatrixCommand) -RegularTestProjectsFile "$(_RegularTestProjectsFile)" - - - - - - From 7767fc070fa5be72fab81d3fd9b32c435e937c5a Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 03:38:52 -0400 Subject: [PATCH 27/48] fixy, and make it quicker --- .github/actions/enumerate-tests/action.yml | 2 +- .../TestEnumerationRunsheetBuilder.targets | 58 ++++++------ eng/Testing.targets | 2 +- eng/scripts/process-test-enumeration.ps1 | 90 +++++++++---------- .../Aspire.Templates.Tests.csproj | 8 +- tests/Directory.Build.targets | 33 +------ 6 files changed, 80 insertions(+), 113 deletions(-) diff --git a/.github/actions/enumerate-tests/action.yml b/.github/actions/enumerate-tests/action.yml index 8b20e4d43f2..a0e1cc3ed9c 100644 --- a/.github/actions/enumerate-tests/action.yml +++ b/.github/actions/enumerate-tests/action.yml @@ -40,7 +40,7 @@ runs: /p:TestsListOutputPath=artifacts/TestsForGithubActions.list /p:TestMatrixOutputPath=artifacts/combined-test-matrix.json /p:BuildOs=${{ runner.os == 'Linux' && 'linux' || runner.os == 'macOS' && 'darwin' || 'windows' }} - -p:PrepareForHelix=true + -p:GenerateCIPartitions=true -bl - name: Generate combined matrix outputs diff --git a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets index a8c1284804f..353eec2a302 100644 --- a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets +++ b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets @@ -12,43 +12,45 @@ test enumeration into the standard Arcade SDK runsheet builder pattern. --> + + + <_ShouldSkipProject>false + <_ShouldSkipProject Condition="$(MSBuildProjectDirectory.Contains('tests\Shared'))">true + <_ShouldSkipProject Condition="$(MSBuildProjectDirectory.Contains('tests\testproject'))">true + <_ShouldSkipProject Condition="$(MSBuildProjectDirectory.Contains('tests\TestingAppHost1'))">true + + + <_ShortName>$([System.IO.Path]::GetFileNameWithoutExtension('$(MSBuildProjectName)').Replace('Aspire.', '').Replace('.Tests', '')) + + - - - - <_ShouldSkipProject>false - <_ShouldSkipProject Condition="$(MSBuildProjectDirectory.Contains('tests\Shared'))">true - <_ShouldSkipProject Condition="$(MSBuildProjectDirectory.Contains('tests\testproject'))">true - <_ShouldSkipProject Condition="$(MSBuildProjectDirectory.Contains('tests\TestingAppHost1'))">true - - - - <_ShortName>$([System.IO.Path]::GetFileNameWithoutExtension('$(MSBuildProjectName)').Replace('Aspire.', '').Replace('.Tests', '')) - + Condition="'$(SkipTests)' != 'true' and '$(_ShouldSkipProject)' != 'true' and '$(IsGitHubActionsRunner)' == 'true' and '$(RunOnGithubActions)' == 'true'"> - - - + + <_CurrentProject Include="$(MSBuildProjectFullPath)" /> + Properties="BuildOs=$(BuildOs)" + Targets="GetRunTestsOnGithubActions"> + Properties="GenerateCIPartitions=true;BuildOs=$(BuildOs)" + Targets="Build;ExtractTestClassNamesV3;WriteTestMetadata" + Condition="'@(_ProjectInfo->Count())' > 0 and '%(_ProjectInfo.SplitTests)' == 'true'" /> + + - + <_RelativeProjectPath>$([System.String]::Copy('$(MSBuildProjectFullPath)').Replace('$(RepoRoot)', '')) @@ -76,21 +78,17 @@ - + + Condition="'@(_ProjectInfo->Count())' > 0" /> - - + Condition="'@(_ProjectInfo->Count())' > 0" /> diff --git a/eng/Testing.targets b/eng/Testing.targets index a4c0e2f4a55..49a86cd0366 100644 --- a/eng/Testing.targets +++ b/eng/Testing.targets @@ -151,7 +151,7 @@ This writes properties like requiresNugets and requiresTestSdk to metadata.json. --> + Condition="'$(GenerateCIPartitions)' == 'true' and '$(IsTestProject)' == 'true'"> <_HelixDir>$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix')) <_MetadataFile>$(_HelixDir)$(MSBuildProjectName).tests.metadata.json diff --git a/eng/scripts/process-test-enumeration.ps1 b/eng/scripts/process-test-enumeration.ps1 index 3d091db44ef..51077c00a9a 100644 --- a/eng/scripts/process-test-enumeration.ps1 +++ b/eng/scripts/process-test-enumeration.ps1 @@ -26,13 +26,18 @@ Write-Host "ArtifactsTmpDir: $ArtifactsTmpDir" $enumerationFiles = Get-ChildItem -Path $ArtifactsTmpDir -Filter '*.testenumeration.json' -ErrorAction SilentlyContinue if (-not $enumerationFiles) { - Write-Host "No test enumeration files found in $ArtifactsTmpDir" + Write-Error "No test enumeration files found in $ArtifactsTmpDir" # Create empty output files "" | Set-Content $TestsListOutputPath if ($TestMatrixOutputPath) { New-Item -Path $TestMatrixOutputPath -ItemType Directory -Force | Out-Null } - exit 0 + exit 1 +} + +if (-not (Test-Path $TestMatrixOutputPath)) { + Write-Error "TestMatrixOutputPath directory does not exist: $TestMatrixOutputPath" + exit 1 } Write-Host "Found $($enumerationFiles.Count) test enumeration files" @@ -89,55 +94,50 @@ else { } -# Generate test matrices if output path is specified -if ($TestMatrixOutputPath) { - Write-Host "Generating test matrices..." - - # Check if TestMatrixOutputPath ends with .json (single file) or is a directory - $isJsonFile = $TestMatrixOutputPath -match '\.json$' - - if ($isJsonFile) { - # Single JSON file output - create directory for intermediate files - $tempMatrixDir = Join-Path (Split-Path $TestMatrixOutputPath -Parent) 'temp-matrix' - New-Item -Path $tempMatrixDir -ItemType Directory -Force | Out-Null - - # Call existing matrix generation script if split tests exist - if ($splitTestProjects.Count -gt 0) { - $matrixScriptPath = Join-Path $RepoRoot 'eng/scripts/generate-test-matrix.ps1' - $testListsDir = Join-Path (Split-Path $TestsListOutputPath -Parent) 'helix' - Write-Host "Calling matrix generation script..." - & $matrixScriptPath -TestListsDirectory $testListsDir -OutputDirectory $tempMatrixDir -BuildOs $BuildOs -RegularTestProjectsFile $TestsListOutputPath - - # Copy the generated matrix file to the expected location - $generatedMatrixFile = Join-Path $tempMatrixDir 'split-tests-matrix.json' - if (Test-Path $generatedMatrixFile) { - Copy-Item $generatedMatrixFile $TestMatrixOutputPath - Write-Host "Matrix file copied to: $TestMatrixOutputPath" - } else { - Write-Warning "Expected matrix file not found at: $generatedMatrixFile" - } +Write-Host "Generating test matrices..." + +# Check if TestMatrixOutputPath ends with .json (single file) or is a directory +$isJsonFile = $TestMatrixOutputPath -match '\.json$' + +if ($isJsonFile) { + # Single JSON file output - create directory for intermediate files + $tempMatrixDir = Join-Path (Split-Path $TestMatrixOutputPath -Parent) 'temp-matrix' + New-Item -Path $tempMatrixDir -ItemType Directory -Force | Out-Null - # Clean up temporary directory - Remove-Item $tempMatrixDir -Recurse -Force -ErrorAction SilentlyContinue + # Call existing matrix generation script if split tests exist + if ($splitTestProjects.Count -gt 0) { + $matrixScriptPath = Join-Path $RepoRoot 'eng/scripts/generate-test-matrix.ps1' + $testListsDir = Join-Path (Split-Path $TestsListOutputPath -Parent) 'helix' + Write-Host "Calling matrix generation script..." + & $matrixScriptPath -TestListsDirectory $testListsDir -OutputDirectory $tempMatrixDir -BuildOs $BuildOs -RegularTestProjectsFile $TestsListOutputPath + + # Copy the generated matrix file to the expected location + $generatedMatrixFile = Join-Path $tempMatrixDir 'split-tests-matrix.json' + if (Test-Path $generatedMatrixFile) { + Copy-Item $generatedMatrixFile $TestMatrixOutputPath + Write-Host "Matrix file copied to: $TestMatrixOutputPath" } else { - # No split tests, create empty matrix - '{"include":[]}' | Set-Content $TestMatrixOutputPath - Write-Host "No split tests found, created empty matrix at: $TestMatrixOutputPath" + Write-Warning "Expected matrix file not found at: $generatedMatrixFile" } - } else { - # Directory output (original behavior) - New-Item -Path $TestMatrixOutputPath -ItemType Directory -Force | Out-Null - # Call existing matrix generation script if split tests exist - if ($splitTestProjects.Count -gt 0) { - $matrixScriptPath = Join-Path $RepoRoot 'eng/scripts/generate-test-matrix.ps1' - $testListsDir = Join-Path (Split-Path $TestsListOutputPath -Parent) 'helix' - Write-Host "Calling matrix generation script..." - & $matrixScriptPath -TestListsDirectory $testListsDir -OutputDirectory $TestMatrixOutputPath -BuildOs $BuildOs -RegularTestProjectsFile $TestsListOutputPath - } + # Clean up temporary directory + Remove-Item $tempMatrixDir -Recurse -Force -ErrorAction SilentlyContinue + } else { + # No split tests, create empty matrix + '{"include":[]}' | Set-Content $TestMatrixOutputPath + Write-Host "No split tests found, created empty matrix at: $TestMatrixOutputPath" } } else { - Write-Host "No TestMatrixOutputPath specified, skipping matrix generation" + # Directory output (original behavior) + New-Item -Path $TestMatrixOutputPath -ItemType Directory -Force | Out-Null + + # Call existing matrix generation script if split tests exist + if ($splitTestProjects.Count -gt 0) { + $matrixScriptPath = Join-Path $RepoRoot 'eng/scripts/generate-test-matrix.ps1' + $testListsDir = Join-Path (Split-Path $TestsListOutputPath -Parent) 'helix' + Write-Host "Calling matrix generation script..." + & $matrixScriptPath -TestListsDirectory $testListsDir -OutputDirectory $TestMatrixOutputPath -BuildOs $BuildOs -RegularTestProjectsFile $TestsListOutputPath + } } Write-Host "Test enumeration processing completed" diff --git a/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj b/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj index 16de814c464..39faf84fdbf 100644 --- a/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj +++ b/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj @@ -13,15 +13,15 @@ true true - - true + true Aspire.Templates.Tests - + + true true true - + 20m 12m diff --git a/tests/Directory.Build.targets b/tests/Directory.Build.targets index 8a9687f856a..5cb99658722 100644 --- a/tests/Directory.Build.targets +++ b/tests/Directory.Build.targets @@ -37,40 +37,9 @@ Overwrite="true" /> - - - - - - - - - - - <_Regex>^\s*($(ExtractTestClassNamesPrefix)[^\($]+) - - - <_TestLines0 Include="$([System.Text.RegularExpressions.Regex]::Match('%(_ListOfTestsLines.Identity)', '$(_Regex)'))" /> - - - - - - - - - - - - Date: Fri, 17 Oct 2025 03:46:40 -0400 Subject: [PATCH 28/48] fix generation --- eng/scripts/process-test-enumeration.ps1 | 35 +++++++++++------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/eng/scripts/process-test-enumeration.ps1 b/eng/scripts/process-test-enumeration.ps1 index 51077c00a9a..a1bbcf1d2c3 100644 --- a/eng/scripts/process-test-enumeration.ps1 +++ b/eng/scripts/process-test-enumeration.ps1 @@ -35,9 +35,20 @@ if (-not $enumerationFiles) { exit 1 } -if (-not (Test-Path $TestMatrixOutputPath)) { - Write-Error "TestMatrixOutputPath directory does not exist: $TestMatrixOutputPath" - exit 1 +# Validate TestMatrixOutputPath if provided +if ($TestMatrixOutputPath) { + # TestMatrixOutputPath must be a JSON file path + if ($TestMatrixOutputPath -notmatch '\.json$') { + Write-Error "TestMatrixOutputPath must be a JSON file path: $TestMatrixOutputPath" + exit 1 + } + + # Check parent directory exists + $parentDir = Split-Path $TestMatrixOutputPath -Parent + if (-not (Test-Path $parentDir)) { + Write-Error "Parent directory for TestMatrixOutputPath does not exist: $parentDir" + exit 1 + } } Write-Host "Found $($enumerationFiles.Count) test enumeration files" @@ -96,11 +107,8 @@ else { Write-Host "Generating test matrices..." -# Check if TestMatrixOutputPath ends with .json (single file) or is a directory -$isJsonFile = $TestMatrixOutputPath -match '\.json$' - -if ($isJsonFile) { - # Single JSON file output - create directory for intermediate files +if ($TestMatrixOutputPath) { + # Create directory for intermediate files $tempMatrixDir = Join-Path (Split-Path $TestMatrixOutputPath -Parent) 'temp-matrix' New-Item -Path $tempMatrixDir -ItemType Directory -Force | Out-Null @@ -127,17 +135,6 @@ if ($isJsonFile) { '{"include":[]}' | Set-Content $TestMatrixOutputPath Write-Host "No split tests found, created empty matrix at: $TestMatrixOutputPath" } -} else { - # Directory output (original behavior) - New-Item -Path $TestMatrixOutputPath -ItemType Directory -Force | Out-Null - - # Call existing matrix generation script if split tests exist - if ($splitTestProjects.Count -gt 0) { - $matrixScriptPath = Join-Path $RepoRoot 'eng/scripts/generate-test-matrix.ps1' - $testListsDir = Join-Path (Split-Path $TestsListOutputPath -Parent) 'helix' - Write-Host "Calling matrix generation script..." - & $matrixScriptPath -TestListsDirectory $testListsDir -OutputDirectory $TestMatrixOutputPath -BuildOs $BuildOs -RegularTestProjectsFile $TestsListOutputPath - } } Write-Host "Test enumeration processing completed" From 5166e2523e7dd26c00ae9ab1879e5282a5d8991a Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 04:40:55 -0400 Subject: [PATCH 29/48] fix --- eng/AfterSolutionBuild.targets | 37 ++++++++- .../TestEnumerationRunsheetBuilder.targets | 49 ++++++++++-- eng/scripts/generate-test-matrix.ps1 | 4 +- eng/scripts/process-test-enumeration.ps1 | 77 +++++++++---------- .../Aspire.Hosting.Tests.csproj | 4 +- 5 files changed, 118 insertions(+), 53 deletions(-) diff --git a/eng/AfterSolutionBuild.targets b/eng/AfterSolutionBuild.targets index 943f173352b..9419d6d2fd7 100644 --- a/eng/AfterSolutionBuild.targets +++ b/eng/AfterSolutionBuild.targets @@ -108,8 +108,41 @@ diff --git a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets index 353eec2a302..d10670fe9b5 100644 --- a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets +++ b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets @@ -1,15 +1,50 @@ diff --git a/eng/scripts/generate-test-matrix.ps1 b/eng/scripts/generate-test-matrix.ps1 index 13d89f82ab2..376183d08de 100644 --- a/eng/scripts/generate-test-matrix.ps1 +++ b/eng/scripts/generate-test-matrix.ps1 @@ -160,7 +160,7 @@ $listFiles = @(Get-ChildItem -Path $TestListsDirectory -Filter '*.tests.list' -R if ($listFiles.Count -eq 0) { $empty = @{ include = @() } New-Item -ItemType Directory -Force -Path $OutputDirectory | Out-Null - $empty | ConvertTo-Json -Depth 5 -Compress | Set-Content -Path (Join-Path $OutputDirectory 'split-tests-matrix.json') -Encoding UTF8 + $empty | ConvertTo-Json -Depth 5 -Compress | Set-Content -Path (Join-Path $OutputDirectory 'combined-tests-matrix.json') -Encoding UTF8 Write-Host "Empty matrix written (no .tests.list files)." exit 0 } @@ -262,5 +262,5 @@ if ($RegularTestProjectsFile -and (Test-Path $RegularTestProjectsFile)) { $matrix = @{ include = $entries } New-Item -ItemType Directory -Force -Path $OutputDirectory | Out-Null -$matrix | ConvertTo-Json -Depth 10 -Compress | Set-Content -Path (Join-Path $OutputDirectory 'split-tests-matrix.json') -Encoding UTF8 +$matrix | ConvertTo-Json -Depth 10 -Compress | Set-Content -Path (Join-Path $OutputDirectory 'combined-tests-matrix.json') -Encoding UTF8 Write-Host "Matrix entries: $($entries.Count)" diff --git a/eng/scripts/process-test-enumeration.ps1 b/eng/scripts/process-test-enumeration.ps1 index a1bbcf1d2c3..6a1a20f17a5 100644 --- a/eng/scripts/process-test-enumeration.ps1 +++ b/eng/scripts/process-test-enumeration.ps1 @@ -36,19 +36,23 @@ if (-not $enumerationFiles) { } # Validate TestMatrixOutputPath if provided -if ($TestMatrixOutputPath) { - # TestMatrixOutputPath must be a JSON file path - if ($TestMatrixOutputPath -notmatch '\.json$') { - Write-Error "TestMatrixOutputPath must be a JSON file path: $TestMatrixOutputPath" - exit 1 - } +# fail if empty +if ($TestMatrixOutputPath -and [string]::IsNullOrWhiteSpace($TestMatrixOutputPath)) { + Write-Error "TestMatrixOutputPath cannot be empty if provided" + exit 1 +} - # Check parent directory exists - $parentDir = Split-Path $TestMatrixOutputPath -Parent - if (-not (Test-Path $parentDir)) { - Write-Error "Parent directory for TestMatrixOutputPath does not exist: $parentDir" - exit 1 - } +# TestMatrixOutputPath must be a JSON file path +if ($TestMatrixOutputPath -notmatch '\.json$') { + Write-Error "TestMatrixOutputPath must be a JSON file path: $TestMatrixOutputPath" + exit 1 +} + +# Check parent directory exists +$parentDir = Split-Path $TestMatrixOutputPath -Parent +if (-not (Test-Path $parentDir)) { + Write-Error "Parent directory for TestMatrixOutputPath does not exist: $parentDir" + exit 1 } Write-Host "Found $($enumerationFiles.Count) test enumeration files" @@ -107,36 +111,29 @@ else { Write-Host "Generating test matrices..." -if ($TestMatrixOutputPath) { - # Create directory for intermediate files - $tempMatrixDir = Join-Path (Split-Path $TestMatrixOutputPath -Parent) 'temp-matrix' - New-Item -Path $tempMatrixDir -ItemType Directory -Force | Out-Null - - # Call existing matrix generation script if split tests exist - if ($splitTestProjects.Count -gt 0) { - $matrixScriptPath = Join-Path $RepoRoot 'eng/scripts/generate-test-matrix.ps1' - $testListsDir = Join-Path (Split-Path $TestsListOutputPath -Parent) 'helix' - Write-Host "Calling matrix generation script..." - & $matrixScriptPath -TestListsDirectory $testListsDir -OutputDirectory $tempMatrixDir -BuildOs $BuildOs -RegularTestProjectsFile $TestsListOutputPath - - # Copy the generated matrix file to the expected location - $generatedMatrixFile = Join-Path $tempMatrixDir 'split-tests-matrix.json' - if (Test-Path $generatedMatrixFile) { - Copy-Item $generatedMatrixFile $TestMatrixOutputPath - Write-Host "Matrix file copied to: $TestMatrixOutputPath" - } else { - Write-Warning "Expected matrix file not found at: $generatedMatrixFile" - } - - # Clean up temporary directory - Remove-Item $tempMatrixDir -Recurse -Force -ErrorAction SilentlyContinue - } else { - # No split tests, create empty matrix - '{"include":[]}' | Set-Content $TestMatrixOutputPath - Write-Host "No split tests found, created empty matrix at: $TestMatrixOutputPath" - } +# Create directory for intermediate files +$tempMatrixDir = Join-Path (Split-Path $TestMatrixOutputPath -Parent) 'temp-matrix' +New-Item -Path $tempMatrixDir -ItemType Directory -Force | Out-Null + +# Call existing matrix generation script if split tests exist +$matrixScriptPath = Join-Path $RepoRoot 'eng/scripts/generate-test-matrix.ps1' +$testListsDir = Join-Path (Split-Path $TestsListOutputPath -Parent) 'helix' +Write-Host "Calling matrix generation script..." +& $matrixScriptPath -TestListsDirectory $testListsDir -OutputDirectory $tempMatrixDir -BuildOs $BuildOs -RegularTestProjectsFile $TestsListOutputPath + +# Copy the generated matrix file to the expected location +$generatedMatrixFile = Join-Path $tempMatrixDir 'combined-tests-matrix.json' +if (Test-Path $generatedMatrixFile) { + Copy-Item $generatedMatrixFile $TestMatrixOutputPath + Write-Host "Matrix file copied to: $TestMatrixOutputPath" +} else { + Write-Error "Expected matrix file not found at: $generatedMatrixFile" + exit 1 } +# Clean up temporary directory +Remove-Item $tempMatrixDir -Recurse -Force -ErrorAction SilentlyContinue + Write-Host "Test enumeration processing completed" Write-Host "Regular projects written to: $TestsListOutputPath" #if ($splitTestProjects.Count -gt 0) { diff --git a/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj b/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj index b3a9e27f224..b2d4d254b01 100644 --- a/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj +++ b/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj @@ -12,9 +12,9 @@ false - true + true Aspire.Hosting.Tests - + 30m 15m From ffae452241dc1a0bec4ad602e61c6fe552548d8a Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 15:27:44 -0400 Subject: [PATCH 30/48] fix per-os runs --- eng/AfterSolutionBuild.targets | 4 +- .../TestEnumerationRunsheetBuilder.targets | 12 +++--- eng/Testing.targets | 38 ++++++++++++++++++- eng/scripts/generate-test-matrix.ps1 | 15 +++++--- eng/scripts/process-test-enumeration.ps1 | 18 ++++----- 5 files changed, 62 insertions(+), 25 deletions(-) diff --git a/eng/AfterSolutionBuild.targets b/eng/AfterSolutionBuild.targets index 9419d6d2fd7..dc30491cea1 100644 --- a/eng/AfterSolutionBuild.targets +++ b/eng/AfterSolutionBuild.targets @@ -151,8 +151,6 @@ - <_BuildOs Condition="'$(BuildOs)' == ''">linux - <_BuildOs Condition="'$(BuildOs)' != ''">$(BuildOs) <_TestsListOutputPath Condition="'$(TestsListOutputPath)' != ''">$([MSBuild]::NormalizePath('$(RepoRoot)', '$(TestsListOutputPath)')) <_TestsListOutputPath Condition="'$(TestsListOutputPath)' == ''">$(ArtifactsDir)/TestsForGithubActions.list <_TestMatrixOutputPath Condition="'$(TestMatrixOutputPath)' != ''">$([MSBuild]::NormalizePath('$(RepoRoot)', '$(TestMatrixOutputPath)')) @@ -160,7 +158,7 @@ - diff --git a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets index d10670fe9b5..c8672ee8b74 100644 --- a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets +++ b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets @@ -67,9 +67,9 @@ <_CurrentProject Include="$(MSBuildProjectFullPath)" /> + + Targets="GetTestOSCompatibility"> @@ -98,14 +98,16 @@ <_HasTestMetadata Condition="Exists('$(_MetadataFile)')">true <_HasTestMetadata Condition="'$(_HasTestMetadata)' != 'true'">false + + <_OsArrayJson>%(_ProjectInfo.SupportedOSes) + <_EnumerationJson>{ "project": "$(MSBuildProjectName)", "fullPath": "$(_RelativeProjectPath)", "shortName": "$(_ShortName)", - "runOnGithubActions": "%(_ProjectInfo.RunTestsOnGithubActions)", + "supportedOSes": [$(_OsArrayJson)], "splitTests": "%(_ProjectInfo.SplitTests)", - "buildOs": "$(BuildOs)", "hasTestMetadata": "$(_HasTestMetadata)", "testListFile": "$(_TestListFile.Replace('$(RepoRoot)', ''))", "metadataFile": "$(_MetadataFile.Replace('$(RepoRoot)', ''))" @@ -121,7 +123,7 @@ Condition="'@(_ProjectInfo->Count())' > 0" /> - diff --git a/eng/Testing.targets b/eng/Testing.targets index 49a86cd0366..de4370e0a16 100644 --- a/eng/Testing.targets +++ b/eng/Testing.targets @@ -146,6 +146,32 @@ + + + + <_ShouldSplit>false + <_ShouldSplit Condition="'$(SplitTestsOnCI)' == 'true'">true + + + <_SupportedOSes> + <_SupportedOSes Condition="'$(RunOnGithubActionsWindows)' == 'true'">$(_SupportedOSes)"windows", + <_SupportedOSes Condition="'$(RunOnGithubActionsLinux)' == 'true'">$(_SupportedOSes)"linux", + <_SupportedOSes Condition="'$(RunOnGithubActionsMacOS)' == 'true'">$(_SupportedOSes)"macos", + + <_SupportedOSes Condition="'$(_SupportedOSes)' != ''">$(_SupportedOSes.TrimEnd(',')) + + + + <_ProjectInfo Include="$(MSBuildProjectFullPath)" + SupportedOSes="$(_SupportedOSes)" + SplitTests="$(_ShouldSplit)" + MSBuildSourceProjectFile="$(MSBuildProjectFullPath)" /> + + + + <_EnablePlaywrightInstall>false <_EnablePlaywrightInstall Condition="'$(EnablePlaywrightInstallForSplitTests)' == 'true'">true + + <_SupportedOSesJson> + <_SupportedOSesJson Condition="'$(RunOnGithubActionsWindows)' == 'true'">$(_SupportedOSesJson)"windows", + <_SupportedOSesJson Condition="'$(RunOnGithubActionsLinux)' == 'true'">$(_SupportedOSesJson)"linux", + <_SupportedOSesJson Condition="'$(RunOnGithubActionsMacOS)' == 'true'">$(_SupportedOSesJson)"macos", + + <_SupportedOSesJson Condition="'$(_SupportedOSesJson)' != ''">$(_SupportedOSesJson.TrimEnd(',')) + <_TestSessionTimeout Condition="'$(SplitTestSessionTimeout)' != ''">$(SplitTestSessionTimeout) <_TestSessionTimeout Condition="'$(SplitTestSessionTimeout)' == ''">20m @@ -187,6 +222,7 @@ "requiresNugets": "$(_RequiresNugets.ToLowerInvariant())", "requiresTestSdk": "$(_RequiresTestSdk.ToLowerInvariant())", "enablePlaywrightInstall": "$(_EnablePlaywrightInstall.ToLowerInvariant())", + "supportedOSes": [$(_SupportedOSesJson)], "testSessionTimeout": "$(_TestSessionTimeout)", "testHangTimeout": "$(_TestHangTimeout)", "uncollectedTestsSessionTimeout": "15m", @@ -200,7 +236,7 @@ Lines="$(_MetadataJson)" Overwrite="true" /> - + diff --git a/eng/scripts/generate-test-matrix.ps1 b/eng/scripts/generate-test-matrix.ps1 index 376183d08de..1f5bd53091d 100644 --- a/eng/scripts/generate-test-matrix.ps1 +++ b/eng/scripts/generate-test-matrix.ps1 @@ -42,9 +42,6 @@ param( [Parameter(Mandatory=$true)] [string]$OutputDirectory, [Parameter(Mandatory=$false)] - [ValidateSet('windows','linux','darwin','')] - [string]$BuildOs = '', - [Parameter(Mandatory=$false)] [string]$RegularTestProjectsFile = '' ) @@ -63,6 +60,7 @@ function Read-Metadata($file, $projectName) { testHangTimeout = '10m' uncollectedTestsSessionTimeout = '15m' uncollectedTestsHangTimeout = '10m' + supportedOSes = @('windows', 'linux', 'macos') } if (-not (Test-Path $file)) { return $defaults } try { @@ -71,7 +69,7 @@ function Read-Metadata($file, $projectName) { $defaults[$k] = $json.$k } } catch { - Write-Warning "Failed parsing metadata for ${projectName}: $_" + throw "Failed parsing metadata for ${projectName}: $_" } return $defaults } @@ -90,6 +88,7 @@ function New-EntryCollection($c,$meta) { enablePlaywrightInstall = ($meta.enablePlaywrightInstall -eq 'true') testSessionTimeout = $meta.testSessionTimeout testHangTimeout = $meta.testHangTimeout + supportedOSes = $meta.supportedOSes } } @@ -110,6 +109,7 @@ function New-EntryUncollected($collections,$meta) { enablePlaywrightInstall = ($meta.enablePlaywrightInstall -eq 'true') testSessionTimeout = ($meta.uncollectedTestsSessionTimeout ?? $meta.testSessionTimeout) testHangTimeout = ($meta.uncollectedTestsHangTimeout ?? $meta.testHangTimeout) + supportedOSes = $meta.supportedOSes } } @@ -132,6 +132,7 @@ function New-EntryClass($full,$meta) { enablePlaywrightInstall = ($meta.enablePlaywrightInstall -eq 'true') testSessionTimeout = $meta.testSessionTimeout testHangTimeout = $meta.testHangTimeout + supportedOSes = $meta.supportedOSes } } @@ -148,12 +149,12 @@ function New-EntryRegular($shortName) { enablePlaywrightInstall = $false testSessionTimeout = '20m' testHangTimeout = '10m' + supportedOSes = @('windows', 'linux', 'macos') } } if (-not (Test-Path $TestListsDirectory)) { - Write-Warning "Test lists directory not found: $TestListsDirectory" - exit 0 + throw "Test lists directory not found: $TestListsDirectory" } $listFiles = @(Get-ChildItem -Path $TestListsDirectory -Filter '*.tests.list' -Recurse -ErrorAction SilentlyContinue) @@ -231,6 +232,7 @@ if ($RegularTestProjectsFile -and (Test-Path $RegularTestProjectsFile)) { enablePlaywrightInstall = 'false' testSessionTimeout = '20m' testHangTimeout = '10m' + supportedOSes = ($proj.supportedOSes ?? @('windows', 'linux', 'macos')) } Write-Host " Using default metadata for $($proj.project) (no metadata file found at $metadataFile)" } @@ -247,6 +249,7 @@ if ($RegularTestProjectsFile -and (Test-Path $RegularTestProjectsFile)) { enablePlaywrightInstall = ($meta.enablePlaywrightInstall -eq 'true') testSessionTimeout = $meta.testSessionTimeout testHangTimeout = $meta.testHangTimeout + supportedOSes = ($proj.supportedOSes ?? $meta.supportedOSes) } $entries.Add($entry) | Out-Null } diff --git a/eng/scripts/process-test-enumeration.ps1 b/eng/scripts/process-test-enumeration.ps1 index 6a1a20f17a5..81a50e4e0f0 100644 --- a/eng/scripts/process-test-enumeration.ps1 +++ b/eng/scripts/process-test-enumeration.ps1 @@ -1,10 +1,7 @@ #!/usr/bin/env pwsh param( - [Parameter(Mandatory=$true)] - [string]$BuildOs, - - [Parameter(Mandatory=$true)] + [Parameter(Mandatory=$false)] [string]$TestsListOutputPath, [Parameter(Mandatory=$false)] @@ -17,7 +14,7 @@ param( [string]$RepoRoot ) -Write-Host "Processing test enumeration files for BuildOs: $BuildOs" +Write-Host "Processing test enumeration files" Write-Host "TestsListOutputPath: $TestsListOutputPath" Write-Host "TestMatrixOutputPath: $TestMatrixOutputPath" Write-Host "ArtifactsTmpDir: $ArtifactsTmpDir" @@ -65,17 +62,18 @@ foreach ($file in $enumerationFiles) { try { $content = Get-Content -Raw $file.FullName | ConvertFrom-Json - # Filter by BuildOs and eligibility - if ($content.buildOs -eq $BuildOs -and $content.runOnGithubActions -eq 'true') { + # Include all test projects that support at least one OS + if ($content.supportedOSes -and $content.supportedOSes.Count -gt 0) { if ($content.splitTests -eq 'true') { $splitTestProjects += $content.shortName } else { # Store full enumeration data for regular tests $regularTestProjects += $content } - Write-Host " Included: $($content.shortName) (Split: $($content.splitTests))" + $osesStr = $content.supportedOSes -join ', ' + Write-Host " Included: $($content.shortName) (OSes: $osesStr, Split: $($content.splitTests))" } else { - Write-Host " Excluded: $($content.shortName) (BuildOs: $($content.buildOs), RunOnGithubActions: $($content.runOnGithubActions))" + Write-Host " Excluded: $($content.shortName) (No supported OSes)" } } catch { @@ -119,7 +117,7 @@ New-Item -Path $tempMatrixDir -ItemType Directory -Force | Out-Null $matrixScriptPath = Join-Path $RepoRoot 'eng/scripts/generate-test-matrix.ps1' $testListsDir = Join-Path (Split-Path $TestsListOutputPath -Parent) 'helix' Write-Host "Calling matrix generation script..." -& $matrixScriptPath -TestListsDirectory $testListsDir -OutputDirectory $tempMatrixDir -BuildOs $BuildOs -RegularTestProjectsFile $TestsListOutputPath +& $matrixScriptPath -TestListsDirectory $testListsDir -OutputDirectory $tempMatrixDir -RegularTestProjectsFile $TestsListOutputPath # Copy the generated matrix file to the expected location $generatedMatrixFile = Join-Path $tempMatrixDir 'combined-tests-matrix.json' From 4292b86c7334ee7e95e4337a959a7b0d496b60e7 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 15:41:43 -0400 Subject: [PATCH 31/48] per-os fixes --- .github/actions/enumerate-tests/action.yml | 9 ++ .github/workflows/tests.yml | 100 ++------------------- eng/scripts/generate-test-matrix.ps1 | 25 +++++- 3 files changed, 40 insertions(+), 94 deletions(-) diff --git a/.github/actions/enumerate-tests/action.yml b/.github/actions/enumerate-tests/action.yml index a0e1cc3ed9c..64f160f46c5 100644 --- a/.github/actions/enumerate-tests/action.yml +++ b/.github/actions/enumerate-tests/action.yml @@ -56,6 +56,15 @@ runs: $testsNotRequiringNugets = @() foreach ($test in $matrixContent.include) { + # Add OS runner mapping to each entry + $osRunner = switch ($test.os) { + 'windows' { 'windows-latest' } + 'linux' { 'ubuntu-latest' } + 'macos' { 'macos-latest' } + default { 'ubuntu-latest' } + } + $test | Add-Member -NotePropertyName 'runs-on' -NotePropertyValue $osRunner -Force + if ($test.requiresNugets -eq "true" -or $test.requiresNugets -eq $true) { $testsRequiringNugets += $test } else { diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c8c22725b3e..4e60f65a5e9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -30,9 +30,9 @@ jobs: with: versionOverrideArg: ${{ inputs.versionOverrideArg }} - tests_no_nugets_lin: + tests_no_nugets: uses: ./.github/workflows/run-tests.yml - name: Tests Linux (No Nugets) + name: Tests (No Nugets) needs: setup_for_tests if: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets).include[0] != null }} strategy: @@ -40,7 +40,7 @@ jobs: matrix: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets) }} with: testShortName: ${{ matrix.shortname }} - os: "ubuntu-latest" + os: ${{ matrix.runs-on }} testProjectPath: ${{ matrix.testProjectPath }} testSessionTimeout: ${{ matrix.testSessionTimeout }} testHangTimeout: ${{ matrix.testHangTimeout }} @@ -50,88 +50,8 @@ jobs: requiresTestSdk: ${{ matrix.requiresTestSdk }} enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} - tests_no_nugets_macos: - uses: ./.github/workflows/run-tests.yml - name: Tests macOS (No Nugets) - needs: setup_for_tests - if: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets).include[0] != null }} - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets) }} - with: - testShortName: ${{ matrix.shortname }} - os: "macos-latest" - testProjectPath: ${{ matrix.testProjectPath }} - testSessionTimeout: ${{ matrix.testSessionTimeout }} - testHangTimeout: ${{ matrix.testHangTimeout }} - extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}" - versionOverrideArg: ${{ inputs.versionOverrideArg }} - requiresNugets: ${{ matrix.requiresNugets }} - requiresTestSdk: ${{ matrix.requiresTestSdk }} - enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} - - tests_no_nugets_win: - uses: ./.github/workflows/run-tests.yml - name: Tests Windows (No Nugets) - needs: setup_for_tests - if: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets).include[0] != null }} - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets) }} - with: - testShortName: ${{ matrix.shortname }} - os: "windows-latest" - testProjectPath: ${{ matrix.testProjectPath }} - testSessionTimeout: ${{ matrix.testSessionTimeout }} - testHangTimeout: ${{ matrix.testHangTimeout }} - extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}" - versionOverrideArg: ${{ inputs.versionOverrideArg }} - requiresNugets: ${{ matrix.requiresNugets }} - requiresTestSdk: ${{ matrix.requiresTestSdk }} - enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} - - tests_requires_nugets_lin: - name: Tests Linux (Requires Nugets) - uses: ./.github/workflows/run-tests.yml - needs: [setup_for_tests, build_packages] - if: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets).include[0] != null }} - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets) }} - with: - testShortName: ${{ matrix.shortname }} - os: "ubuntu-latest" - testProjectPath: ${{ matrix.testProjectPath }} - testSessionTimeout: ${{ matrix.testSessionTimeout }} - testHangTimeout: ${{ matrix.testHangTimeout }} - extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}" - versionOverrideArg: ${{ inputs.versionOverrideArg }} - requiresNugets: ${{ matrix.requiresNugets }} - requiresTestSdk: ${{ matrix.requiresTestSdk }} - enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} - - tests_requires_nugets_macos: - name: Tests macOS (Requires Nugets) - uses: ./.github/workflows/run-tests.yml - needs: [setup_for_tests, build_packages] - if: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets).include[0] != null }} - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets) }} - with: - testShortName: ${{ matrix.shortname }} - os: "macos-latest" - testProjectPath: ${{ matrix.testProjectPath }} - testSessionTimeout: ${{ matrix.testSessionTimeout }} - testHangTimeout: ${{ matrix.testHangTimeout }} - extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}" - versionOverrideArg: ${{ inputs.versionOverrideArg }} - requiresNugets: ${{ matrix.requiresNugets }} - requiresTestSdk: ${{ matrix.requiresTestSdk }} - enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} - - tests_requires_nugets_win: - name: Tests Windows (Requires Nugets) + tests_requires_nugets: + name: Tests (Requires Nugets) uses: ./.github/workflows/run-tests.yml needs: [setup_for_tests, build_packages] if: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets).include[0] != null }} @@ -140,7 +60,7 @@ jobs: matrix: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets) }} with: testShortName: ${{ matrix.shortname }} - os: "windows-latest" + os: ${{ matrix.runs-on }} testProjectPath: ${{ matrix.testProjectPath }} testSessionTimeout: ${{ matrix.testSessionTimeout }} testHangTimeout: ${{ matrix.testHangTimeout }} @@ -181,12 +101,8 @@ jobs: name: Final Test Results needs: [ extension_tests_win, - tests_no_nugets_lin, - tests_no_nugets_macos, - tests_no_nugets_win, - tests_requires_nugets_lin, - tests_requires_nugets_macos, - tests_requires_nugets_win + tests_no_nugets, + tests_requires_nugets ] steps: - name: Checkout code diff --git a/eng/scripts/generate-test-matrix.ps1 b/eng/scripts/generate-test-matrix.ps1 index 1f5bd53091d..5d066500b04 100644 --- a/eng/scripts/generate-test-matrix.ps1 +++ b/eng/scripts/generate-test-matrix.ps1 @@ -263,7 +263,28 @@ if ($RegularTestProjectsFile -and (Test-Path $RegularTestProjectsFile)) { } } -$matrix = @{ include = $entries } +# Expand entries to create one per supported OS +$expandedEntries = [System.Collections.Generic.List[object]]::new() +foreach ($entry in $entries) { + $supportedOSes = $entry.supportedOSes + if (-not $supportedOSes) { + $supportedOSes = @('windows', 'linux', 'macos') + } + + foreach ($os in $supportedOSes) { + $expandedEntry = [ordered]@{} + foreach ($key in $entry.Keys) { + if ($key -ne 'supportedOSes') { + $expandedEntry[$key] = $entry[$key] + } + } + $expandedEntry['os'] = $os + $expandedEntries.Add($expandedEntry) | Out-Null + } +} + +$matrix = @{ include = $expandedEntries } New-Item -ItemType Directory -Force -Path $OutputDirectory | Out-Null $matrix | ConvertTo-Json -Depth 10 -Compress | Set-Content -Path (Join-Path $OutputDirectory 'combined-tests-matrix.json') -Encoding UTF8 -Write-Host "Matrix entries: $($entries.Count)" +Write-Host "Matrix entries (before OS expansion): $($entries.Count)" +Write-Host "Matrix entries (after OS expansion): $($expandedEntries.Count)" From 12a793ef1e43a892a25a4ee01fc8d8a1df8681c3 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 15:47:00 -0400 Subject: [PATCH 32/48] cleanup --- .../TestEnumerationRunsheetBuilder.targets | 2 +- eng/scripts/generate-test-matrix.ps1 | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets index c8672ee8b74..e3e8e24ec2d 100644 --- a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets +++ b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets @@ -124,7 +124,7 @@ diff --git a/eng/scripts/generate-test-matrix.ps1 b/eng/scripts/generate-test-matrix.ps1 index 5d066500b04..e9b6fde6383 100644 --- a/eng/scripts/generate-test-matrix.ps1 +++ b/eng/scripts/generate-test-matrix.ps1 @@ -221,7 +221,7 @@ if ($RegularTestProjectsFile -and (Test-Path $RegularTestProjectsFile)) { $meta = $null if ($metadataFile -and (Test-Path $metadataFile)) { $meta = Read-Metadata $metadataFile $proj.project - Write-Host " Loaded metadata for $($proj.project) from $metadataFile (requiresNugets=$($meta.requiresNugets))" + #Write-Host " Loaded metadata for $($proj.project) from $metadataFile (requiresNugets=$($meta.requiresNugets))" } else { # Use defaults if no metadata file exists $meta = @{ @@ -270,7 +270,7 @@ foreach ($entry in $entries) { if (-not $supportedOSes) { $supportedOSes = @('windows', 'linux', 'macos') } - + foreach ($os in $supportedOSes) { $expandedEntry = [ordered]@{} foreach ($key in $entry.Keys) { From 8e125b322a4c8190e4d3da0c1a7cc319473c5032 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 17:35:36 -0400 Subject: [PATCH 33/48] improve json --- .github/actions/enumerate-tests/action.yml | 59 ++++++-- eng/Testing.targets | 2 +- eng/scripts/generate-test-matrix.ps1 | 165 +++++++++++++-------- 3 files changed, 149 insertions(+), 77 deletions(-) diff --git a/.github/actions/enumerate-tests/action.yml b/.github/actions/enumerate-tests/action.yml index 64f160f46c5..e8ea23a4854 100644 --- a/.github/actions/enumerate-tests/action.yml +++ b/.github/actions/enumerate-tests/action.yml @@ -51,24 +51,61 @@ runs: if (Test-Path $matrixFilePath) { $matrixContent = Get-Content -Raw $matrixFilePath | ConvertFrom-Json - # Split tests based on requiresNugets property + # Define defaults to apply when properties are missing + $defaults = @{ + extraTestArgs = '' + requiresNugets = $false + requiresTestSdk = $false + enablePlaywrightInstall = $false + testSessionTimeout = '20m' + testHangTimeout = '10m' + supportedOSes = @('windows', 'linux', 'macos') + } + + # Split tests based on requiresNugets property and expand by OS $testsRequiringNugets = @() $testsNotRequiringNugets = @() foreach ($test in $matrixContent.include) { - # Add OS runner mapping to each entry - $osRunner = switch ($test.os) { - 'windows' { 'windows-latest' } - 'linux' { 'ubuntu-latest' } - 'macos' { 'macos-latest' } - default { 'ubuntu-latest' } + # Apply defaults for missing properties + foreach ($key in $defaults.Keys) { + if ($key -ne 'supportedOSes' -and -not $test.PSObject.Properties.Name.Contains($key)) { + $test | Add-Member -NotePropertyName $key -NotePropertyValue $defaults[$key] -Force + } } - $test | Add-Member -NotePropertyName 'runs-on' -NotePropertyValue $osRunner -Force - if ($test.requiresNugets -eq "true" -or $test.requiresNugets -eq $true) { - $testsRequiringNugets += $test + # Get supported OSes (use test-specific if present, otherwise default) + $supportedOSes = if ($test.PSObject.Properties.Name.Contains('supportedOSes')) { + $test.supportedOSes } else { - $testsNotRequiringNugets += $test + $defaults.supportedOSes + } + + # Expand test entry for each supported OS + foreach ($os in $supportedOSes) { + # Create a new entry for each OS + $testCopy = [PSCustomObject]@{} + foreach ($prop in $test.PSObject.Properties) { + if ($prop.Name -ne 'supportedOSes') { + $testCopy | Add-Member -NotePropertyName $prop.Name -NotePropertyValue $prop.Value -Force + } + } + + # Add OS runner mapping + $osRunner = switch ($os) { + 'windows' { 'windows-latest' } + 'linux' { 'ubuntu-latest' } + 'macos' { 'macos-latest' } + default { 'ubuntu-latest' } + } + $testCopy | Add-Member -NotePropertyName 'runs-on' -NotePropertyValue $osRunner -Force + + # Add to appropriate list based on requiresNugets + if ($testCopy.requiresNugets -eq "true" -or $testCopy.requiresNugets -eq $true) { + $testsRequiringNugets += $testCopy + } else { + $testsNotRequiringNugets += $testCopy + } } } diff --git a/eng/Testing.targets b/eng/Testing.targets index de4370e0a16..8400c6665b5 100644 --- a/eng/Testing.targets +++ b/eng/Testing.targets @@ -236,7 +236,7 @@ Lines="$(_MetadataJson)" Overwrite="true" /> - + diff --git a/eng/scripts/generate-test-matrix.ps1 b/eng/scripts/generate-test-matrix.ps1 index e9b6fde6383..3fef7295cf4 100644 --- a/eng/scripts/generate-test-matrix.ps1 +++ b/eng/scripts/generate-test-matrix.ps1 @@ -48,11 +48,23 @@ param( $ErrorActionPreference = 'Stop' Set-StrictMode -Version Latest +# Define default values - only include properties in output when they differ from these +$script:Defaults = @{ + extraTestArgs = '' + requiresNugets = $false + requiresTestSdk = $false + enablePlaywrightInstall = $false + testSessionTimeout = '20m' + testHangTimeout = '10m' + supportedOSes = @('windows', 'linux', 'macos') +} + function Read-Metadata($file, $projectName) { $defaults = @{ projectName = $projectName testClassNamesPrefix = $projectName testProjectPath = "tests/$projectName/$projectName.csproj" + extraTestArgs = '' requiresNugets = 'false' requiresTestSdk = 'false' enablePlaywrightInstall = 'false' @@ -74,22 +86,45 @@ function Read-Metadata($file, $projectName) { return $defaults } +function Add-OptionalProperty($entry, $key, $value, $default) { + # Only add property if it differs from the default + if ($null -ne $default) { + if ($value -is [Array] -and $default -is [Array]) { + # Compare arrays + if (($value.Count -ne $default.Count) -or (Compare-Object $value $default)) { + $entry[$key] = $value + } + } elseif ($value -ne $default) { + $entry[$key] = $value + } + } else { + # No default, always include + $entry[$key] = $value + } +} + function New-EntryCollection($c,$meta) { $projectShortName = $meta.projectName -replace '^Aspire\.' -replace '\.Tests$' - [ordered]@{ + $extraTestArgsValue = "--filter-trait `"Partition=$c`"" + + $entry = [ordered]@{ type = 'collection' projectName = $meta.projectName name = $c shortname = "${projectShortName}_$c" testProjectPath = $meta.testProjectPath - extraTestArgs = "--filter-trait `"Partition=$c`"" - requiresNugets = ($meta.requiresNugets -eq 'true') - requiresTestSdk = ($meta.requiresTestSdk -eq 'true') - enablePlaywrightInstall = ($meta.enablePlaywrightInstall -eq 'true') - testSessionTimeout = $meta.testSessionTimeout - testHangTimeout = $meta.testHangTimeout - supportedOSes = $meta.supportedOSes } + + # Add optional properties only if they differ from defaults + Add-OptionalProperty $entry 'extraTestArgs' $extraTestArgsValue $script:Defaults.extraTestArgs + Add-OptionalProperty $entry 'requiresNugets' ($meta.requiresNugets -eq 'true') $script:Defaults.requiresNugets + Add-OptionalProperty $entry 'requiresTestSdk' ($meta.requiresTestSdk -eq 'true') $script:Defaults.requiresTestSdk + Add-OptionalProperty $entry 'enablePlaywrightInstall' ($meta.enablePlaywrightInstall -eq 'true') $script:Defaults.enablePlaywrightInstall + Add-OptionalProperty $entry 'testSessionTimeout' $meta.testSessionTimeout $script:Defaults.testSessionTimeout + Add-OptionalProperty $entry 'testHangTimeout' $meta.testHangTimeout $script:Defaults.testHangTimeout + Add-OptionalProperty $entry 'supportedOSes' $meta.supportedOSes $script:Defaults.supportedOSes + + return $entry } function New-EntryUncollected($collections,$meta) { @@ -97,20 +132,30 @@ function New-EntryUncollected($collections,$meta) { foreach ($c in $collections) { $filters += "--filter-not-trait `"Partition=$c`"" } - [ordered]@{ + $extraTestArgsValue = ($filters -join ' ') + + $entry = [ordered]@{ type = 'uncollected' projectName = $meta.projectName name = 'UncollectedTests' shortname = 'Uncollected' testProjectPath = $meta.testProjectPath - extraTestArgs = ($filters -join ' ') - requiresNugets = ($meta.requiresNugets -eq 'true') - requiresTestSdk = ($meta.requiresTestSdk -eq 'true') - enablePlaywrightInstall = ($meta.enablePlaywrightInstall -eq 'true') - testSessionTimeout = ($meta.uncollectedTestsSessionTimeout ?? $meta.testSessionTimeout) - testHangTimeout = ($meta.uncollectedTestsHangTimeout ?? $meta.testHangTimeout) - supportedOSes = $meta.supportedOSes } + + # Add optional properties only if they differ from defaults + # Note: uncollected tests may have different timeout defaults + $uncollectedSessionTimeout = $meta.uncollectedTestsSessionTimeout ?? $meta.testSessionTimeout + $uncollectedHangTimeout = $meta.uncollectedTestsHangTimeout ?? $meta.testHangTimeout + + Add-OptionalProperty $entry 'extraTestArgs' $extraTestArgsValue $script:Defaults.extraTestArgs + Add-OptionalProperty $entry 'requiresNugets' ($meta.requiresNugets -eq 'true') $script:Defaults.requiresNugets + Add-OptionalProperty $entry 'requiresTestSdk' ($meta.requiresTestSdk -eq 'true') $script:Defaults.requiresTestSdk + Add-OptionalProperty $entry 'enablePlaywrightInstall' ($meta.enablePlaywrightInstall -eq 'true') $script:Defaults.enablePlaywrightInstall + Add-OptionalProperty $entry 'testSessionTimeout' $uncollectedSessionTimeout $script:Defaults.testSessionTimeout + Add-OptionalProperty $entry 'testHangTimeout' $uncollectedHangTimeout $script:Defaults.testHangTimeout + Add-OptionalProperty $entry 'supportedOSes' $meta.supportedOSes $script:Defaults.supportedOSes + + return $entry } function New-EntryClass($full,$meta) { @@ -119,41 +164,43 @@ function New-EntryClass($full,$meta) { if ($prefix -and $full.StartsWith("$prefix.")) { $short = $full.Substring($prefix.Length + 1) } - [ordered]@{ + $extraTestArgsValue = "--filter-class `"$full`"" + + $entry = [ordered]@{ type = 'class' projectName = $meta.projectName name = $short shortname = $short fullClassName = $full testProjectPath = $meta.testProjectPath - extraTestArgs = "--filter-class `"$full`"" - requiresNugets = ($meta.requiresNugets -eq 'true') - requiresTestSdk = ($meta.requiresTestSdk -eq 'true') - enablePlaywrightInstall = ($meta.enablePlaywrightInstall -eq 'true') - testSessionTimeout = $meta.testSessionTimeout - testHangTimeout = $meta.testHangTimeout - supportedOSes = $meta.supportedOSes } + + # Add optional properties only if they differ from defaults + Add-OptionalProperty $entry 'extraTestArgs' $extraTestArgsValue $script:Defaults.extraTestArgs + Add-OptionalProperty $entry 'requiresNugets' ($meta.requiresNugets -eq 'true') $script:Defaults.requiresNugets + Add-OptionalProperty $entry 'requiresTestSdk' ($meta.requiresTestSdk -eq 'true') $script:Defaults.requiresTestSdk + Add-OptionalProperty $entry 'enablePlaywrightInstall' ($meta.enablePlaywrightInstall -eq 'true') $script:Defaults.enablePlaywrightInstall + Add-OptionalProperty $entry 'testSessionTimeout' $meta.testSessionTimeout $script:Defaults.testSessionTimeout + Add-OptionalProperty $entry 'testHangTimeout' $meta.testHangTimeout $script:Defaults.testHangTimeout + Add-OptionalProperty $entry 'supportedOSes' $meta.supportedOSes $script:Defaults.supportedOSes + + return $entry } function New-EntryRegular($shortName) { - [ordered]@{ + $entry = [ordered]@{ type = 'regular' projectName = "Aspire.$shortName.Tests" name = $shortName shortname = $shortName testProjectPath = "tests/Aspire.$shortName.Tests/Aspire.$shortName.Tests.csproj" - extraTestArgs = "" - requiresNugets = $false - requiresTestSdk = $false - enablePlaywrightInstall = $false - testSessionTimeout = '20m' - testHangTimeout = '10m' - supportedOSes = @('windows', 'linux', 'macos') } -} -if (-not (Test-Path $TestListsDirectory)) { + # All defaults match, so no need to add any optional properties + # (extraTestArgs is empty, which matches the default) + + return $entry +}if (-not (Test-Path $TestListsDirectory)) { throw "Test lists directory not found: $TestListsDirectory" } @@ -224,15 +271,18 @@ if ($RegularTestProjectsFile -and (Test-Path $RegularTestProjectsFile)) { #Write-Host " Loaded metadata for $($proj.project) from $metadataFile (requiresNugets=$($meta.requiresNugets))" } else { # Use defaults if no metadata file exists + # Note: supportedOSes comes from the project JSON, not defaults + $projectSupportedOSes = if ($proj.PSObject.Properties['supportedOSes']) { $proj.supportedOSes } else { @('windows', 'linux', 'macos') } $meta = @{ projectName = $proj.project testProjectPath = $proj.fullPath + extraTestArgs = '' requiresNugets = 'false' requiresTestSdk = 'false' enablePlaywrightInstall = 'false' testSessionTimeout = '20m' testHangTimeout = '10m' - supportedOSes = ($proj.supportedOSes ?? @('windows', 'linux', 'macos')) + supportedOSes = $projectSupportedOSes } Write-Host " Using default metadata for $($proj.project) (no metadata file found at $metadataFile)" } @@ -243,14 +293,20 @@ if ($RegularTestProjectsFile -and (Test-Path $RegularTestProjectsFile)) { name = $proj.shortName shortname = $proj.shortName testProjectPath = $proj.fullPath - extraTestArgs = "" - requiresNugets = ($meta.requiresNugets -eq 'true') - requiresTestSdk = ($meta.requiresTestSdk -eq 'true') - enablePlaywrightInstall = ($meta.enablePlaywrightInstall -eq 'true') - testSessionTimeout = $meta.testSessionTimeout - testHangTimeout = $meta.testHangTimeout - supportedOSes = ($proj.supportedOSes ?? $meta.supportedOSes) } + + # Add optional properties only if they differ from defaults + # Note: supportedOSes from the project JSON takes precedence + $finalSupportedOSes = if ($proj.PSObject.Properties['supportedOSes']) { $proj.supportedOSes } else { $meta.supportedOSes } + + Add-OptionalProperty $entry 'extraTestArgs' $meta.extraTestArgs $script:Defaults.extraTestArgs + Add-OptionalProperty $entry 'requiresNugets' ($meta.requiresNugets -eq 'true') $script:Defaults.requiresNugets + Add-OptionalProperty $entry 'requiresTestSdk' ($meta.requiresTestSdk -eq 'true') $script:Defaults.requiresTestSdk + Add-OptionalProperty $entry 'enablePlaywrightInstall' ($meta.enablePlaywrightInstall -eq 'true') $script:Defaults.enablePlaywrightInstall + Add-OptionalProperty $entry 'testSessionTimeout' $meta.testSessionTimeout $script:Defaults.testSessionTimeout + Add-OptionalProperty $entry 'testHangTimeout' $meta.testHangTimeout $script:Defaults.testHangTimeout + Add-OptionalProperty $entry 'supportedOSes' $finalSupportedOSes $script:Defaults.supportedOSes + $entries.Add($entry) | Out-Null } } else { @@ -263,28 +319,7 @@ if ($RegularTestProjectsFile -and (Test-Path $RegularTestProjectsFile)) { } } -# Expand entries to create one per supported OS -$expandedEntries = [System.Collections.Generic.List[object]]::new() -foreach ($entry in $entries) { - $supportedOSes = $entry.supportedOSes - if (-not $supportedOSes) { - $supportedOSes = @('windows', 'linux', 'macos') - } - - foreach ($os in $supportedOSes) { - $expandedEntry = [ordered]@{} - foreach ($key in $entry.Keys) { - if ($key -ne 'supportedOSes') { - $expandedEntry[$key] = $entry[$key] - } - } - $expandedEntry['os'] = $os - $expandedEntries.Add($expandedEntry) | Out-Null - } -} - -$matrix = @{ include = $expandedEntries } +$matrix = @{ include = $entries } New-Item -ItemType Directory -Force -Path $OutputDirectory | Out-Null $matrix | ConvertTo-Json -Depth 10 -Compress | Set-Content -Path (Join-Path $OutputDirectory 'combined-tests-matrix.json') -Encoding UTF8 -Write-Host "Matrix entries (before OS expansion): $($entries.Count)" -Write-Host "Matrix entries (after OS expansion): $($expandedEntries.Count)" +Write-Host "Matrix entries: $($entries.Count)" From dc37ade48abd7d33013aedd5f2e19c69ffd6268f Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 18:18:14 -0400 Subject: [PATCH 34/48] wip --- eng/AfterSolutionBuild.targets | 22 ++++++---- eng/scripts/extract-test-metadata.ps1 | 63 +++++++++++++++++---------- tests/Directory.Build.targets | 35 ++++----------- 3 files changed, 61 insertions(+), 59 deletions(-) diff --git a/eng/AfterSolutionBuild.targets b/eng/AfterSolutionBuild.targets index dc30491cea1..e683d7e849f 100644 --- a/eng/AfterSolutionBuild.targets +++ b/eng/AfterSolutionBuild.targets @@ -123,12 +123,11 @@ * Location of test metadata files (for split tests) 2. Aggregation Phase (this target): - - Calls process-test-enumeration.ps1 to: + - Calls build-test-matrix.ps1 to: * Find all .testenumeration.json files in ArtifactsTmpDir - * Filter by BuildOs and runOnGithubActions flag - * Separate regular vs split test projects - * Call generate-test-matrix-for-split-tests.ps1 - * Generate combined test matrix JSON + * Filter by BuildOs + * Process both regular and split test projects + * Generate combined test matrix JSON in a single pass 3. Output: - TestMatrixOutputPath: Combined JSON matrix with all test entries @@ -154,14 +153,19 @@ <_TestsListOutputPath Condition="'$(TestsListOutputPath)' != ''">$([MSBuild]::NormalizePath('$(RepoRoot)', '$(TestsListOutputPath)')) <_TestsListOutputPath Condition="'$(TestsListOutputPath)' == ''">$(ArtifactsDir)/TestsForGithubActions.list <_TestMatrixOutputPath Condition="'$(TestMatrixOutputPath)' != ''">$([MSBuild]::NormalizePath('$(RepoRoot)', '$(TestMatrixOutputPath)')) - <_ProcessScript>$([MSBuild]::NormalizePath($(RepoRoot), 'eng', 'scripts', 'process-test-enumeration.ps1')) + <_BuildMatrixScript>$([MSBuild]::NormalizePath($(RepoRoot), 'eng', 'scripts', 'build-test-matrix.ps1')) + + + <_CurrentOS Condition="$([MSBuild]::IsOSPlatform('Linux'))">linux + <_CurrentOS Condition="$([MSBuild]::IsOSPlatform('Windows'))">windows + <_CurrentOS Condition="$([MSBuild]::IsOSPlatform('OSX'))">macos - - + - + diff --git a/eng/scripts/extract-test-metadata.ps1 b/eng/scripts/extract-test-metadata.ps1 index eb9598fece9..3b55314d01d 100644 --- a/eng/scripts/extract-test-metadata.ps1 +++ b/eng/scripts/extract-test-metadata.ps1 @@ -6,7 +6,7 @@ Determines splitting mode by extracting Collection and Trait attributes from the test assembly: - Uses ExtractTestPartitions tool to find [Collection("name")] or [Trait("Partition", "name")] attributes - If partitions found → partition mode (collections) - - Else → class mode + - Else → class mode (runs --list-tests to enumerate classes) Outputs a .tests.list file with either: collection:Name ... @@ -17,12 +17,13 @@ Also updates the per-project metadata JSON with mode and collections. -.PARAMETER TestAssemblyOutputFile - Path to a temporary file containing the raw --list-tests output (one line per entry). - .PARAMETER TestAssemblyPath Path to the test assembly DLL for extracting partition attributes. +.PARAMETER RunCommand + The command to run the test assembly (e.g., "dotnet exec "). + Only invoked if partition extraction fails and class-based splitting is needed. + .PARAMETER TestClassNamesPrefix Namespace prefix used to recognize test classes (e.g. Aspire.Templates.Tests). @@ -41,15 +42,16 @@ .NOTES PowerShell 7+ Fails fast if zero test classes discovered when in class mode. + Optimized to only run --list-tests when partition extraction fails. #> [CmdletBinding()] param( [Parameter(Mandatory=$true)] - [string]$TestAssemblyOutputFile, + [string]$TestAssemblyPath, [Parameter(Mandatory=$true)] - [string]$TestAssemblyPath, + [string]$RunCommand, [Parameter(Mandatory=$true)] [string]$TestClassNamesPrefix, @@ -70,12 +72,10 @@ param( $ErrorActionPreference = 'Stop' Set-StrictMode -Version Latest -if (-not (Test-Path $TestAssemblyOutputFile)) { - Write-Error "TestAssemblyOutputFile not found: $TestAssemblyOutputFile" +if (-not (Test-Path $TestAssemblyPath)) { + Write-Error "TestAssemblyPath not found: $TestAssemblyPath" } -$raw = Get-Content -LiteralPath $TestAssemblyOutputFile -ErrorAction Stop - $collections = [System.Collections.Generic.HashSet[string]]::new() $classes = [System.Collections.Generic.HashSet[string]]::new() @@ -133,18 +133,7 @@ finally { } } -# Extract class names from test listing -$classNamePattern = '^(\s*)' + [Regex]::Escape($TestClassNamesPrefix) + '\.([^\.]+)\.' - -foreach ($line in $raw) { - # Extract class name from test name - # Format: " Namespace.ClassName.MethodName(...)" or "Namespace.ClassName.MethodName" - if ($line -match $classNamePattern) { - $className = "$TestClassNamesPrefix.$($Matches[2])" - $classes.Add($className) | Out-Null - } -} - +# Apply collection filtering $skipList = @() if ($TestCollectionsToSkip) { $skipList = $TestCollectionsToSkip -split ';' | ForEach-Object { $_.Trim() } | Where-Object { $_ } @@ -152,10 +141,36 @@ if ($TestCollectionsToSkip) { $filteredCollections = @($collections | Where-Object { $skipList -notcontains $_ }) +# Determine mode: if we have partitions, use collection mode; otherwise fall back to class mode $mode = if ($filteredCollections.Count -gt 0) { 'collection' } else { 'class' } -if ($classes.Count -eq 0 -and $mode -eq 'class') { - Write-Error "No test classes discovered matching prefix '$TestClassNamesPrefix'." +# Only run --list-tests if we need class-based splitting (no partitions found) +if ($mode -eq 'class') { + Write-Host "No partitions found. Running --list-tests to extract class names..." + + # Run the test assembly with --list-tests to get all test names + $testOutput = & $RunCommand --filter-not-trait category=failing --list-tests 2>&1 + + if ($LASTEXITCODE -ne 0) { + Write-Warning "Test listing command failed with exit code $LASTEXITCODE. Attempting to parse partial output..." + } + + # Extract class names from test listing + $classNamePattern = '^(\s*)' + [Regex]::Escape($TestClassNamesPrefix) + '\.([^\.]+)\.' + + foreach ($line in $testOutput) { + $lineStr = $line.ToString() + # Extract class name from test name + # Format: " Namespace.ClassName.MethodName(...)" or "Namespace.ClassName.MethodName" + if ($lineStr -match $classNamePattern) { + $className = "$TestClassNamesPrefix.$($Matches[2])" + $classes.Add($className) | Out-Null + } + } + + if ($classes.Count -eq 0) { + Write-Error "No test classes discovered matching prefix '$TestClassNamesPrefix'." + } } $outputDir = [System.IO.Path]::GetDirectoryName($OutputListFile) diff --git a/tests/Directory.Build.targets b/tests/Directory.Build.targets index 5cb99658722..2b77b28907c 100644 --- a/tests/Directory.Build.targets +++ b/tests/Directory.Build.targets @@ -47,20 +47,15 @@ - - - - - <_DiscoveryScriptPath>$(RepoRoot)eng\scripts\extract-test-metadata.ps1 <_TestListFile>$(TestArchiveTestsDir)$(MSBuildProjectName).tests.list + <_TestListFileAbs>$([MSBuild]::NormalizePath('$(RepoRoot)', '$(_TestListFile)')) <_MetadataFile>$(TestArchiveTestsDir)$(MSBuildProjectName).tests.metadata.json + <_MetadataFileAbs>$([MSBuild]::NormalizePath('$(RepoRoot)', '$(_MetadataFile)')) <_RelativeProjectPath>$(MSBuildProjectDirectory.Replace('$(RepoRoot)', '')) @@ -74,15 +69,6 @@ - - - <_TempOutputFile>$(TestArchiveTestsDir)$(MSBuildProjectName).tests.output.tmp - - - - <_InitialMetadataLines Include="{" /> @@ -99,11 +85,11 @@ <_InitialMetadataLines Include="}" /> - - + @@ -111,12 +97,12 @@ <_PwshCommand Condition="'$(OS)' == 'Windows_NT'">powershell <_TestAssemblyPath>$(TargetDir)$(TargetFileName) <_DiscoveryCommand>$(_PwshCommand) -NoProfile -ExecutionPolicy Bypass -File "$(_DiscoveryScriptPath)" - <_DiscoveryCommand>$(_DiscoveryCommand) -TestAssemblyOutputFile "$(_TempOutputFile)" <_DiscoveryCommand>$(_DiscoveryCommand) -TestAssemblyPath "$(_TestAssemblyPath)" + <_DiscoveryCommand>$(_DiscoveryCommand) -RunCommand "$(RunCommand)" <_DiscoveryCommand>$(_DiscoveryCommand) -TestClassNamesPrefix "$(TestClassNamesPrefix)" <_DiscoveryCommand Condition="'$(_CollectionsToSkip)' != ''">$(_DiscoveryCommand) -TestCollectionsToSkip "$(_CollectionsToSkip)" - <_DiscoveryCommand>$(_DiscoveryCommand) -OutputListFile "$(_TestListFile)" - <_DiscoveryCommand>$(_DiscoveryCommand) -MetadataJsonFile "$(_MetadataFile)" + <_DiscoveryCommand>$(_DiscoveryCommand) -OutputListFile "$(_TestListFileAbs)" + <_DiscoveryCommand>$(_DiscoveryCommand) -MetadataJsonFile "$(_MetadataFileAbs)" <_DiscoveryCommand>$(_DiscoveryCommand) -RepoRoot "$(RepoRoot)" @@ -124,12 +110,9 @@ IgnoreExitCode="false" WorkingDirectory="$(RepoRoot)" /> - - - - + From bcb5068798c12001b7fa9335a8eaa865e90f141e Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 18:23:54 -0400 Subject: [PATCH 35/48] consolidated --- eng/scripts/build-test-matrix.ps1 | 328 ++++++++++++++++++++++++++++++ 1 file changed, 328 insertions(+) create mode 100644 eng/scripts/build-test-matrix.ps1 diff --git a/eng/scripts/build-test-matrix.ps1 b/eng/scripts/build-test-matrix.ps1 new file mode 100644 index 00000000000..c1ebae69842 --- /dev/null +++ b/eng/scripts/build-test-matrix.ps1 @@ -0,0 +1,328 @@ +<# +.SYNOPSIS + Builds the combined test matrix for GitHub Actions from test enumeration files. + +.DESCRIPTION + This script consolidates the functionality of process-test-enumeration.ps1 and + generate-test-matrix.ps1 into a single optimized script that: + 1. Collects all .testenumeration.json files + 2. Filters tests by supported OSes + 3. Separates regular tests from split tests + 4. Generates matrix entries for all tests (with partition/class splitting) + 5. Writes the final combined-tests-matrix.json in a single pass + + No intermediate files are created - all data processing happens in memory. + +.PARAMETER ArtifactsTmpDir + Directory containing .testenumeration.json files from test projects. + +.PARAMETER ArtifactsHelixDir + Directory containing .tests.list and .tests.metadata.json files. + +.PARAMETER OutputMatrixFile + Path to write the combined test matrix JSON file. + +.PARAMETER TestsListOutputFile + Optional path to write backward-compatible test list file (regular tests only). + +.PARAMETER CurrentOS + Current operating system (linux, windows, macos). Filters tests by supported OSes. + +.NOTES + PowerShell 7+ + Replaces: process-test-enumeration.ps1 + generate-test-matrix.ps1 +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory=$true)] + [string]$ArtifactsTmpDir, + + [Parameter(Mandatory=$true)] + [string]$ArtifactsHelixDir, + + [Parameter(Mandatory=$true)] + [string]$OutputMatrixFile, + + [Parameter(Mandatory=$false)] + [string]$TestsListOutputFile = "", + + [Parameter(Mandatory=$true)] + [string]$CurrentOS +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +# Normalize OS name +$CurrentOS = $CurrentOS.ToLowerInvariant() + +Write-Host "Building test matrix for OS: $CurrentOS" +Write-Host "Enumerations directory: $ArtifactsTmpDir" +Write-Host "Helix directory: $ArtifactsHelixDir" + +# Helper function to create matrix entry for regular (non-split) tests +function New-RegularTestEntry { + param( + [Parameter(Mandatory=$true)] + $Enumeration, + [Parameter(Mandatory=$false)] + $Metadata = $null + ) + + $entry = [ordered]@{ + type = 'regular' + project = $Enumeration.project + shortname = $Enumeration.shortName + testProjectPath = $Enumeration.fullPath + workitemprefix = $Enumeration.project + } + + # Add metadata if available + if ($Metadata) { + if ($Metadata.testSessionTimeout) { $entry['testSessionTimeout'] = $Metadata.testSessionTimeout } + if ($Metadata.testHangTimeout) { $entry['testHangTimeout'] = $Metadata.testHangTimeout } + if ($Metadata.requiresNugets -eq 'true') { $entry['requiresNugets'] = 'true' } + if ($Metadata.requiresTestSdk -eq 'true') { $entry['requiresTestSdk'] = 'true' } + if ($Metadata.enablePlaywrightInstall -eq 'true') { $entry['enablePlaywrightInstall'] = 'true' } + } + + # Add supported OSes + $entry['supportedOSes'] = @($Enumeration.supportedOSes) + + return $entry +} + +# Helper function to create matrix entry for collection-based split tests +function New-CollectionTestEntry { + param( + [Parameter(Mandatory=$true)] + [string]$CollectionName, + [Parameter(Mandatory=$true)] + $Metadata, + [Parameter(Mandatory=$true)] + [bool]$IsUncollected + ) + + $suffix = if ($IsUncollected) { 'uncollected' } else { $CollectionName } + + $entry = [ordered]@{ + type = 'collection' + project = $Metadata.projectName + shortname = "$($Metadata.projectName)-$suffix" + testProjectPath = $Metadata.testProjectPath + workitemprefix = "$($Metadata.projectName)_$suffix" + collection = $CollectionName + } + + # Use uncollected timeouts if available, otherwise use regular + if ($IsUncollected) { + if ($Metadata.uncollectedTestsSessionTimeout) { + $entry['testSessionTimeout'] = $Metadata.uncollectedTestsSessionTimeout + } elseif ($Metadata.testSessionTimeout) { + $entry['testSessionTimeout'] = $Metadata.testSessionTimeout + } + + if ($Metadata.uncollectedTestsHangTimeout) { + $entry['testHangTimeout'] = $Metadata.uncollectedTestsHangTimeout + } elseif ($Metadata.testHangTimeout) { + $entry['testHangTimeout'] = $Metadata.testHangTimeout + } + } else { + if ($Metadata.testSessionTimeout) { $entry['testSessionTimeout'] = $Metadata.testSessionTimeout } + if ($Metadata.testHangTimeout) { $entry['testHangTimeout'] = $Metadata.testHangTimeout } + } + + if ($Metadata.requiresNugets -eq 'true') { $entry['requiresNugets'] = 'true' } + if ($Metadata.requiresTestSdk -eq 'true') { $entry['requiresTestSdk'] = 'true' } + if ($Metadata.enablePlaywrightInstall -eq 'true') { $entry['enablePlaywrightInstall'] = 'true' } + + # Add test filter for collection-based splitting + if ($IsUncollected) { + $entry['extraTestArgs'] = '--filter-not-trait "Partition"' + } else { + $entry['extraTestArgs'] = "--filter-trait `"Partition=$CollectionName`"" + } + + # Add supported OSes from metadata (should match enumeration) + if ($Metadata.supportedOSes) { + $entry['supportedOSes'] = @($Metadata.supportedOSes) + } + + return $entry +} + +# Helper function to create matrix entry for class-based split tests +function New-ClassTestEntry { + param( + [Parameter(Mandatory=$true)] + [string]$ClassName, + [Parameter(Mandatory=$true)] + $Metadata + ) + + # Extract short class name (last segment after last dot) + $shortClassName = $ClassName.Split('.')[-1] + + $entry = [ordered]@{ + type = 'class' + project = $Metadata.projectName + shortname = "$($Metadata.projectName)-$shortClassName" + testProjectPath = $Metadata.testProjectPath + workitemprefix = "$($Metadata.projectName)_$shortClassName" + classname = $ClassName + } + + if ($Metadata.testSessionTimeout) { $entry['testSessionTimeout'] = $Metadata.testSessionTimeout } + if ($Metadata.testHangTimeout) { $entry['testHangTimeout'] = $Metadata.testHangTimeout } + if ($Metadata.requiresNugets -eq 'true') { $entry['requiresNugets'] = 'true' } + if ($Metadata.requiresTestSdk -eq 'true') { $entry['requiresTestSdk'] = 'true' } + if ($Metadata.enablePlaywrightInstall -eq 'true') { $entry['enablePlaywrightInstall'] = 'true' } + + # Add test filter for class-based splitting + $entry['extraTestArgs'] = "--filter-class `"$ClassName`"" + + # Add supported OSes from metadata + if ($Metadata.supportedOSes) { + $entry['supportedOSes'] = @($Metadata.supportedOSes) + } + + return $entry +} + +# 1. Collect all enumeration files +$enumerationFiles = @(Get-ChildItem -Path $ArtifactsTmpDir -Filter "*.testenumeration.json" -ErrorAction SilentlyContinue) + +if ($enumerationFiles.Count -eq 0) { + Write-Warning "No test enumeration files found in $ArtifactsTmpDir" + # Create empty matrix + $matrix = @{ include = @() } + $matrix | ConvertTo-Json -Depth 10 -Compress | Set-Content -Path $OutputMatrixFile + Write-Host "Created empty test matrix: $OutputMatrixFile" + exit 0 +} + +Write-Host "Found $($enumerationFiles.Count) test enumeration file(s)" + +# 2. Build matrix entries +$matrixEntries = [System.Collections.Generic.List[object]]::new() +$regularTestsList = [System.Collections.Generic.List[string]]::new() + +foreach ($enumFile in $enumerationFiles) { + $enum = Get-Content -Raw -Path $enumFile.FullName | ConvertFrom-Json + + Write-Host "Processing: $($enum.project)" + + # Filter by supported OSes (skip if current OS not supported) + if ($enum.supportedOSes -and $enum.supportedOSes.Count -gt 0) { + $osSupported = $false + foreach ($os in $enum.supportedOSes) { + if ($os.ToLowerInvariant() -eq $CurrentOS) { + $osSupported = $true + break + } + } + + if (-not $osSupported) { + Write-Host " ⊘ Skipping (not supported on $CurrentOS)" + continue + } + } + + # Check if this is a split test with metadata + if ($enum.splitTests -eq 'true' -and $enum.hasTestMetadata -eq 'true') { + Write-Host " → Split test (processing partitions/classes)" + + # Read metadata and test list + $metaFile = Join-Path $ArtifactsHelixDir "$($enum.project).tests.metadata.json" + $listFile = Join-Path $ArtifactsHelixDir "$($enum.project).tests.list" + + if (-not (Test-Path $metaFile)) { + Write-Warning " ⚠ Metadata file not found: $metaFile" + continue + } + + if (-not (Test-Path $listFile)) { + Write-Warning " ⚠ Test list file not found: $listFile" + continue + } + + $metadata = Get-Content -Raw -Path $metaFile | ConvertFrom-Json + + # Add supported OSes to metadata from enumeration + $metadata | Add-Member -Force -MemberType NoteProperty -Name 'supportedOSes' -Value $enum.supportedOSes + + $listLines = Get-Content -Path $listFile + + $partitionCount = 0 + $classCount = 0 + + foreach ($line in $listLines) { + $line = $line.Trim() + if ([string]::IsNullOrWhiteSpace($line)) { continue } + + if ($line -match '^collection:(.+)$') { + # Collection/partition entry + $collectionName = $Matches[1] + $entry = New-CollectionTestEntry -CollectionName $collectionName -Metadata $metadata -IsUncollected $false + $matrixEntries.Add($entry) + $partitionCount++ + } + elseif ($line -match '^uncollected:\*$') { + # Uncollected tests entry + $entry = New-CollectionTestEntry -CollectionName '*' -Metadata $metadata -IsUncollected $true + $matrixEntries.Add($entry) + $partitionCount++ + } + elseif ($line -match '^class:(.+)$') { + # Class-based entry + $className = $Matches[1] + $entry = New-ClassTestEntry -ClassName $className -Metadata $metadata + $matrixEntries.Add($entry) + $classCount++ + } + } + + Write-Host " ✓ Added $partitionCount partition(s) and $classCount class(es)" + } + else { + # Regular (non-split) test + Write-Host " → Regular test" + + # Try to load metadata if available + $metaFile = Join-Path $ArtifactsHelixDir "$($enum.project).tests.metadata.json" + $metadata = $null + if (Test-Path $metaFile) { + $metadata = Get-Content -Raw -Path $metaFile | ConvertFrom-Json + } + + $entry = New-RegularTestEntry -Enumeration $enum -Metadata $metadata + $matrixEntries.Add($entry) + $regularTestsList.Add($enum.shortName) + + Write-Host " ✓ Added regular test" + } +} + +# 3. Write final matrix +Write-Host "" +Write-Host "Generated $($matrixEntries.Count) total matrix entries" + +$matrix = @{ include = $matrixEntries } +$outputDir = [System.IO.Path]::GetDirectoryName($OutputMatrixFile) +if ($outputDir -and -not (Test-Path $outputDir)) { + New-Item -ItemType Directory -Path $outputDir -Force | Out-Null +} + +$matrix | ConvertTo-Json -Depth 10 -Compress | Set-Content -Path $OutputMatrixFile -Encoding UTF8 + +Write-Host "✓ Matrix written to: $OutputMatrixFile" + +# 4. Write backward-compatible test list if requested +if ($TestsListOutputFile) { + $regularTestsList | Set-Content -Path $TestsListOutputFile -Encoding UTF8 + Write-Host "✓ Test list written to: $TestsListOutputFile" +} + +Write-Host "" +Write-Host "Matrix build complete!" From 690bf562d0c53f220b5285cd87a3be42e3ce44db Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 19:08:26 -0400 Subject: [PATCH 36/48] cleanup --- eng/AfterSolutionBuild.targets | 14 +++-- .../TestEnumerationRunsheetBuilder.targets | 4 +- eng/Testing.targets | 4 +- eng/scripts/build-test-matrix.ps1 | 57 +++++++++++++++---- 4 files changed, 61 insertions(+), 18 deletions(-) diff --git a/eng/AfterSolutionBuild.targets b/eng/AfterSolutionBuild.targets index e683d7e849f..c391b2fa6ad 100644 --- a/eng/AfterSolutionBuild.targets +++ b/eng/AfterSolutionBuild.targets @@ -155,14 +155,18 @@ <_TestMatrixOutputPath Condition="'$(TestMatrixOutputPath)' != ''">$([MSBuild]::NormalizePath('$(RepoRoot)', '$(TestMatrixOutputPath)')) <_BuildMatrixScript>$([MSBuild]::NormalizePath($(RepoRoot), 'eng', 'scripts', 'build-test-matrix.ps1')) - - <_CurrentOS Condition="$([MSBuild]::IsOSPlatform('Linux'))">linux - <_CurrentOS Condition="$([MSBuild]::IsOSPlatform('Windows'))">windows - <_CurrentOS Condition="$([MSBuild]::IsOSPlatform('OSX'))">macos + + <_HelixDir>$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix')) + + + <_CurrentOS Condition="'$(BuildOs)' != ''">$(BuildOs) + <_CurrentOS Condition="'$(BuildOs)' == '' and $([MSBuild]::IsOSPlatform('Linux'))">all + <_CurrentOS Condition="'$(BuildOs)' == '' and $([MSBuild]::IsOSPlatform('Windows'))">all + <_CurrentOS Condition="'$(BuildOs)' == '' and $([MSBuild]::IsOSPlatform('OSX'))">all - diff --git a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets index e3e8e24ec2d..b84640de143 100644 --- a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets +++ b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets @@ -92,7 +92,9 @@ <_RelativeProjectPath Condition="$(_RelativeProjectPath.StartsWith('/')) or $(_RelativeProjectPath.StartsWith('\'))">$(_RelativeProjectPath.Substring(1)) - <_HelixDir>$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix')) + + <_HelixDir Condition="'$(TestArchiveTestsDir)' != ''">$(TestArchiveTestsDir) + <_HelixDir Condition="'$(TestArchiveTestsDir)' == ''">$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix')) <_TestListFile>$(_HelixDir)$(MSBuildProjectName).tests.list <_MetadataFile>$(_HelixDir)$(MSBuildProjectName).tests.metadata.json <_HasTestMetadata Condition="Exists('$(_MetadataFile)')">true diff --git a/eng/Testing.targets b/eng/Testing.targets index 8400c6665b5..11b1f92598b 100644 --- a/eng/Testing.targets +++ b/eng/Testing.targets @@ -179,7 +179,9 @@ - <_HelixDir>$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix')) + + <_HelixDir Condition="'$(TestArchiveTestsDir)' != ''">$(TestArchiveTestsDir) + <_HelixDir Condition="'$(TestArchiveTestsDir)' == ''">$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix')) <_MetadataFile>$(_HelixDir)$(MSBuildProjectName).tests.metadata.json diff --git a/eng/scripts/build-test-matrix.ps1 b/eng/scripts/build-test-matrix.ps1 index c1ebae69842..40a914759cb 100644 --- a/eng/scripts/build-test-matrix.ps1 +++ b/eng/scripts/build-test-matrix.ps1 @@ -27,6 +27,7 @@ .PARAMETER CurrentOS Current operating system (linux, windows, macos). Filters tests by supported OSes. + If not specified or set to 'all', includes tests for all OSes without filtering. .NOTES PowerShell 7+ @@ -47,8 +48,8 @@ param( [Parameter(Mandatory=$false)] [string]$TestsListOutputFile = "", - [Parameter(Mandatory=$true)] - [string]$CurrentOS + [Parameter(Mandatory=$false)] + [string]$CurrentOS = "all" ) $ErrorActionPreference = 'Stop' @@ -57,7 +58,14 @@ Set-StrictMode -Version Latest # Normalize OS name $CurrentOS = $CurrentOS.ToLowerInvariant() -Write-Host "Building test matrix for OS: $CurrentOS" +# Determine if we should filter by OS +$filterByOS = $CurrentOS -ne 'all' + +if ($filterByOS) { + Write-Host "Building test matrix for OS: $CurrentOS" +} else { + Write-Host "Building combined test matrix for all OSes" +} Write-Host "Enumerations directory: $ArtifactsTmpDir" Write-Host "Helix directory: $ArtifactsHelixDir" @@ -214,7 +222,8 @@ foreach ($enumFile in $enumerationFiles) { Write-Host "Processing: $($enum.project)" # Filter by supported OSes (skip if current OS not supported) - if ($enum.supportedOSes -and $enum.supportedOSes.Count -gt 0) { + # Only filter if a specific OS was requested + if ($filterByOS -and $enum.supportedOSes -and $enum.supportedOSes.Count -gt 0) { $osSupported = $false foreach ($os in $enum.supportedOSes) { if ($os.ToLowerInvariant() -eq $CurrentOS) { @@ -233,9 +242,26 @@ foreach ($enumFile in $enumerationFiles) { if ($enum.splitTests -eq 'true' -and $enum.hasTestMetadata -eq 'true') { Write-Host " → Split test (processing partitions/classes)" - # Read metadata and test list - $metaFile = Join-Path $ArtifactsHelixDir "$($enum.project).tests.metadata.json" - $listFile = Join-Path $ArtifactsHelixDir "$($enum.project).tests.list" + # Read metadata and test list - use paths from enumeration file if available + if ($enum.metadataFile) { + # Path is relative to repo root, make it absolute + $metaFile = Join-Path $PSScriptRoot "../../$($enum.metadataFile)" -Resolve -ErrorAction SilentlyContinue + if (-not $metaFile) { + $metaFile = [System.IO.Path]::Combine($PSScriptRoot, "../..", $enum.metadataFile) + } + } else { + $metaFile = Join-Path $ArtifactsHelixDir "$($enum.project).tests.metadata.json" + } + + if ($enum.testListFile) { + # Path is relative to repo root, make it absolute + $listFile = Join-Path $PSScriptRoot "../../$($enum.testListFile)" -Resolve -ErrorAction SilentlyContinue + if (-not $listFile) { + $listFile = [System.IO.Path]::Combine($PSScriptRoot, "../..", $enum.testListFile) + } + } else { + $listFile = Join-Path $ArtifactsHelixDir "$($enum.project).tests.list" + } if (-not (Test-Path $metaFile)) { Write-Warning " ⚠ Metadata file not found: $metaFile" @@ -287,10 +313,19 @@ foreach ($enumFile in $enumerationFiles) { } else { # Regular (non-split) test - Write-Host " → Regular test" + #Write-Host " → Regular test" + + # Try to load metadata if available - use path from enumeration file if available + if ($enum.metadataFile) { + # Path is relative to repo root, make it absolute + $metaFile = Join-Path $PSScriptRoot "../../$($enum.metadataFile)" -Resolve -ErrorAction SilentlyContinue + if (-not $metaFile) { + $metaFile = [System.IO.Path]::Combine($PSScriptRoot, "../..", $enum.metadataFile) + } + } else { + $metaFile = Join-Path $ArtifactsHelixDir "$($enum.project).tests.metadata.json" + } - # Try to load metadata if available - $metaFile = Join-Path $ArtifactsHelixDir "$($enum.project).tests.metadata.json" $metadata = $null if (Test-Path $metaFile) { $metadata = Get-Content -Raw -Path $metaFile | ConvertFrom-Json @@ -300,7 +335,7 @@ foreach ($enumFile in $enumerationFiles) { $matrixEntries.Add($entry) $regularTestsList.Add($enum.shortName) - Write-Host " ✓ Added regular test" + #Write-Host " ✓ Added regular test" } } From 0286960a1e9b36c0e50e2f7be5f3315e833a996e Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 19:19:29 -0400 Subject: [PATCH 37/48] cleanup --- docs/test-splitting/IMPLEMENTATION_PLAN.md | 203 ---- docs/test-splitting/IMPLEMENTATION_PLAN_V2.md | 384 -------- docs/test-splitting/IMPLEMENTATION_PLAN_V3.md | 332 ------- docs/test-splitting/IMPLEMENTATION_SUMMARY.md | 255 ------ .../STEP_01_DISCOVERY_HELPER.md | 368 -------- .../test-splitting/STEP_01_MSBUILD_TARGETS.md | 236 ----- .../STEP_01_MSBUILD_TARGETS_V2.md | 295 ------ .../STEP_02_MSBUILD_TARGETS_V3.md | 446 --------- .../STEP_02_POWERSHELL_SCRIPT.md | 374 -------- .../STEP_02_POWERSHELL_SCRIPT_V2.md | 708 -------------- docs/test-splitting/STEP_03_GITHUB_ACTIONS.md | 414 --------- .../STEP_03_MATRIX_GENERATOR_V3.md | 865 ------------------ docs/test-splitting/STEP_04_PROJECT_CONFIG.md | 230 ----- .../STEP_04_PROJECT_CONFIG_V2.md | 490 ---------- .../STEP_04_PROJECT_CONFIG_V3.md | 316 ------- docs/test-splitting/STEP_05_TESTING_V3.md | 373 -------- docs/test-splitting/STEP_06_CI_INTEGRATION.md | 318 ------- .../TestEnumerationRunsheetBuilder.targets | 18 +- eng/scripts/generate-test-matrix.ps1 | 325 ------- eng/scripts/process-test-enumeration.ps1 | 142 --- 20 files changed, 9 insertions(+), 7083 deletions(-) delete mode 100644 docs/test-splitting/IMPLEMENTATION_PLAN.md delete mode 100644 docs/test-splitting/IMPLEMENTATION_PLAN_V2.md delete mode 100644 docs/test-splitting/IMPLEMENTATION_PLAN_V3.md delete mode 100644 docs/test-splitting/IMPLEMENTATION_SUMMARY.md delete mode 100644 docs/test-splitting/STEP_01_DISCOVERY_HELPER.md delete mode 100644 docs/test-splitting/STEP_01_MSBUILD_TARGETS.md delete mode 100644 docs/test-splitting/STEP_01_MSBUILD_TARGETS_V2.md delete mode 100644 docs/test-splitting/STEP_02_MSBUILD_TARGETS_V3.md delete mode 100644 docs/test-splitting/STEP_02_POWERSHELL_SCRIPT.md delete mode 100644 docs/test-splitting/STEP_02_POWERSHELL_SCRIPT_V2.md delete mode 100644 docs/test-splitting/STEP_03_GITHUB_ACTIONS.md delete mode 100644 docs/test-splitting/STEP_03_MATRIX_GENERATOR_V3.md delete mode 100644 docs/test-splitting/STEP_04_PROJECT_CONFIG.md delete mode 100644 docs/test-splitting/STEP_04_PROJECT_CONFIG_V2.md delete mode 100644 docs/test-splitting/STEP_04_PROJECT_CONFIG_V3.md delete mode 100644 docs/test-splitting/STEP_05_TESTING_V3.md delete mode 100644 docs/test-splitting/STEP_06_CI_INTEGRATION.md delete mode 100644 eng/scripts/generate-test-matrix.ps1 delete mode 100644 eng/scripts/process-test-enumeration.ps1 diff --git a/docs/test-splitting/IMPLEMENTATION_PLAN.md b/docs/test-splitting/IMPLEMENTATION_PLAN.md deleted file mode 100644 index a66f75d464d..00000000000 --- a/docs/test-splitting/IMPLEMENTATION_PLAN.md +++ /dev/null @@ -1,203 +0,0 @@ -# Test Splitting Implementation Plan for dotnet/aspire - -**Date**: 2025-10-16 -**Author**: @radical -**Objective**: Implement a unified, MSBuild-based test splitting mechanism that works across all 3 OSes (Linux, macOS, Windows) and both CI systems (GitHub Actions, Azure DevOps). - -## Overview - -This plan implements automatic test partitioning by class for long-running test projects. The mechanism: -- ✅ Works on all 3 OSes (Linux, macOS, Windows) -- ✅ Works on GitHub Actions and Azure DevOps -- ✅ Uses MSBuild + PowerShell for deterministic, version-controlled matrix generation -- ✅ Allows simple opt-in via project properties -- ✅ Maintains backward compatibility with existing non-split tests - -## Current State - -### Existing Split Tests -- **Aspire.Templates.Tests**: Already uses class-based splitting -- Splits into ~10-15 test classes -- Each OS generates its own matrix (separate setup jobs) -- Uses `--filter-class` to run individual classes - -### Problem Statement -3-4 test projects have very long run times: -1. **Aspire.Hosting.Tests** - Very long, needs splitting -2. Likely other Hosting-related tests -3. Some integration test projects - -Currently only Templates.Tests uses splitting. We need a **common mechanism** that: -- Any test project can opt into -- Automatically handles class enumeration -- Generates appropriate matrices for all OSes -- Requires minimal YAML changes - -## Architecture - -### Component Overview - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ GitHub Actions │ -├─────────────────────────────────────────────────────────────────┤ -│ │ -│ ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ │ -│ │ setup_for_ │ │ setup_for_ │ │ setup_for_ │ │ -│ │ tests_lin │ │ tests_macos │ │ tests_win │ │ -│ │ (ubuntu) │ │ (macos) │ │ (windows) │ │ -│ └───────┬───────┘ └───────┬───────┘ └───────┬───────┘ │ -│ │ │ │ │ -│ └─────────┬────────┴────────┬─────────┘ │ -│ ▼ ▼ │ -│ ┌───────────────────────────────────┐ │ -│ │ .github/actions/enumerate-tests │ │ -│ └───────────────┬───────────────────┘ │ -│ ▼ │ -│ ┌───────────────────────────────────┐ │ -│ │ tests/Shared/GetTestProjects.proj │ │ -│ │ (MSBuild orchestration) │ │ -│ └───────────────┬───────────────────┘ │ -│ │ │ -│ ┌──────────────────┼──────────────────┐ │ -│ ▼ ▼ ▼ │ -│ ┌─────────────┐ ┌──────────────┐ ┌───────────────┐ │ -│ │ Regular │ │ Build Split │ │ Generate │ │ -│ │ Tests List │ │ Test Projects│ │ Matrices │ │ -│ └─────────────┘ └──────┬───────┘ └───────┬───────┘ │ -│ │ │ │ -│ ▼ ▼ │ -│ ┌────────────────────────────────────┐ │ -│ │ eng/scripts/generate-test-matrix.ps1│ │ -│ │ (PowerShell - reads .tests.list │ │ -│ │ and .metadata.json files) │ │ -│ └────────────────┬───────────────────┘ │ -│ ▼ │ -│ ┌────────────────────────────────┐ │ -│ │ artifacts/test-matrices/ │ │ -│ │ split-tests-matrix.json │ │ -│ └────────────────────────────────┘ │ -│ │ -└─────────────────────────────────────────────────────────────────┘ -``` - -### Data Flow - -``` -Per-OS Setup Job - ↓ -enumerate-tests action - ↓ -GetTestProjects.proj (MSBuild) - ↓ - ├─→ Regular Tests → .list file - │ - └─→ Split Tests Projects → .list.split-projects - ↓ - Build each split project - ↓ - ExtractTestClassNames target - ↓ - Generate per-project: - ├─→ ProjectName.tests.list (test class names) - └─→ ProjectName.tests.metadata.json (config) - ↓ - generate-test-matrix.ps1 - ↓ - split-tests-matrix.json - ↓ - GitHub Actions matrix -``` - -## Implementation Steps - -See individual files: -1. [Step 1: MSBuild Targets](./STEP_01_MSBUILD_TARGETS.md) -2. [Step 2: PowerShell Script](./STEP_02_POWERSHELL_SCRIPT.md) -3. [Step 3: GitHub Actions](./STEP_03_GITHUB_ACTIONS.md) -4. [Step 4: Project Configuration](./STEP_04_PROJECT_CONFIG.md) -5. [Step 5: Testing & Validation](./STEP_05_TESTING.md) - -## OS-Specific Considerations - -### Per-OS Matrix Generation - -Each OS generates its own matrix in parallel: -- **Linux** (ubuntu-latest): `setup_for_tests_lin` -- **macOS** (macos-latest): `setup_for_tests_macos` -- **Windows** (windows-latest): `setup_for_tests_win` - -This is critical because: -1. Projects can opt-in/out per OS via `RunOnGithubActions{Windows|Linux|MacOS}` properties -2. File paths differ (slash direction) -3. Some tests only run on specific OSes (e.g., Docker on Linux) - -### PowerShell Cross-Platform - -The `generate-test-matrix.ps1` script: -- ✅ Uses PowerShell Core features (cross-platform) -- ✅ Uses `System.IO.Path.Combine()` for path handling -- ✅ Avoids OS-specific cmdlets -- ✅ Tested on all 3 OSes - -## Migration Strategy - -### Phase 1: Infrastructure (Week 1) -- Implement MSBuild targets -- Create PowerShell script -- Update enumerate-tests action -- Test with Aspire.Templates.Tests (already splitting) - -### Phase 2: Enable for Long-Running Tests (Week 2) -- Migrate Aspire.Templates.Tests to new mechanism -- Enable splitting for Aspire.Hosting.Tests -- Enable for 2-3 other long-running projects -- Monitor CI times - -### Phase 3: Optimization (Week 3) -- Analyze job distribution -- Fine-tune timeouts -- Add any missing metadata fields -- Document usage - -## Success Criteria - -- ✅ All OSes generate correct matrices -- ✅ Split tests run in parallel per class -- ✅ Regular tests continue to work unchanged -- ✅ CI time for long-running projects reduced by 50%+ -- ✅ No increase in flakiness -- ✅ Works on both GitHub Actions and Azure DevOps - -## Rollback Plan - -If issues arise: -1. Set `SplitTestsForCI=false` in problematic project -2. Project reverts to regular single-job execution -3. No YAML changes needed (matrix will be empty) - -## Files Modified/Created - -### New Files -- `eng/scripts/generate-test-matrix.ps1` -- `docs/testing/test-splitting.md` (documentation) - -### Modified Files -- `tests/Directory.Build.targets` -- `tests/Shared/GetTestProjects.proj` -- `.github/actions/enumerate-tests/action.yml` -- `.github/workflows/tests.yml` -- `tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj` -- `tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj` (if enabled) - -## Next Steps - -1. Review this plan -2. Begin implementation following step-by-step guides -3. Create PR with Phase 1 changes -4. Test thoroughly on all OSes -5. Gradually roll out to long-running projects - ---- - -**Implementation Details**: See individual step markdown files in this directory. \ No newline at end of file diff --git a/docs/test-splitting/IMPLEMENTATION_PLAN_V2.md b/docs/test-splitting/IMPLEMENTATION_PLAN_V2.md deleted file mode 100644 index 7ee175d2f1a..00000000000 --- a/docs/test-splitting/IMPLEMENTATION_PLAN_V2.md +++ /dev/null @@ -1,384 +0,0 @@ -# Test Splitting Implementation Plan v2 - Hybrid Collection + Class Splitting - -**Date**: 2025-10-16 -**Author**: @radical -**Objective**: Implement a flexible test splitting mechanism that supports: -- ✅ Individual jobs per xUnit Collection (for grouped tests) -- ✅ ONE job for all uncollected tests (catch-all) -- ✅ Works across all 3 OSes (Linux, macOS, Windows) - -## Overview - -This v2 plan enhances the original with **hybrid collection-based splitting**: - -### Splitting Strategies - -``` -Test Project - ├─ Tests with [Collection("Group1")] → 1 job (all Group1 tests) - ├─ Tests with [Collection("Group2")] → 1 job (all Group2 tests) - └─ All other tests (no collection) → 1 job (ClassA + ClassB + ClassC + ...) -``` - -### Example with 3 Jobs - -``` -Aspire.Hosting.Tests - ├─ [Collection("SlowDatabaseTests")] → Job 1: Collection_SlowDatabaseTests - ├─ [Collection("IntegrationTests")] → Job 2: Collection_IntegrationTests - └─ QuickTests, FastTests, UnitTests... → Job 3: UncollectedTests - (no collection attribute) -``` - -**Total**: 3 parallel jobs instead of 1 monolithic job - -### xUnit Collection Features Used - -- `[Collection("name")]` attribute to group test classes -- `--filter-collection ` to run specific collection -- `--filter-not-collection --filter-not-collection ` to run everything NOT in collections - -## Architecture Changes - -### Test Discovery Output Format - -The `.tests.list` file now includes collections discovered: - -``` -# Format: : -collection:SlowDatabaseTests -collection:IntegrationTests -uncollected:* -``` - -Note: We don't list individual classes anymore - just collections + one uncollected entry. - -### Matrix Entry Structure - -```json -{ - "include": [ - { - "type": "collection", - "name": "SlowDatabaseTests", - "filterArg": "--filter-collection SlowDatabaseTests", - "shortname": "Collection_SlowDatabaseTests", - "testSessionTimeout": "30m", - "testHangTimeout": "15m" - }, - { - "type": "collection", - "name": "IntegrationTests", - "filterArg": "--filter-collection IntegrationTests", - "shortname": "Collection_IntegrationTests", - "testSessionTimeout": "25m", - "testHangTimeout": "12m" - }, - { - "type": "uncollected", - "name": "UncollectedTests", - "filterArg": "--filter-not-collection SlowDatabaseTests --filter-not-collection IntegrationTests", - "shortname": "Uncollected", - "testSessionTimeout": "20m", - "testHangTimeout": "10m" - } - ] -} -``` - -## Key Benefits - -### Efficiency -- **Fewer jobs**: Only create jobs for collections + 1 catch-all -- **Less overhead**: No job-per-class overhead for fast tests -- **Better resource usage**: Group related tests with shared fixtures - -### Flexibility -- **Opt-in granularity**: Only split out slow/problematic test groups -- **Simple default**: Tests without collections run normally together -- **Developer control**: Use `[Collection]` to optimize as needed - -### Backward Compatible -- **No collections?** → 1 job (current behavior) -- **All collections?** → N jobs (one per collection) -- **Mixed?** → N+1 jobs (collections + uncollected) - -## Implementation Steps - -See updated files: -1. [Step 1: MSBuild Targets (v2)](./STEP_01_MSBUILD_TARGETS_V2.md) -2. [Step 2: PowerShell Script (v2)](./STEP_02_POWERSHELL_SCRIPT_V2.md) -3. [Step 3: GitHub Actions (No Changes)](./STEP_03_GITHUB_ACTIONS.md) -4. [Step 4: Project Configuration (v2)](./STEP_04_PROJECT_CONFIG_V2.md) -5. [Step 5: Testing & Validation (v2)](./STEP_05_TESTING_V2.md) - -## Usage Examples - -### Example 1: No Collections (Simple Case) - -```xml - - true - Aspire.Hosting.Tests - -``` - -```csharp -// No collection attributes -public class QuickTests { } -public class FastTests { } -public class UnitTests { } -``` - -**Result**: 1 job running all tests (equivalent to not splitting) - -### Example 2: Hybrid Splitting (Recommended) - -```xml - - true - Aspire.Hosting.Tests - -``` - -```csharp -// Slow database tests - group together -[Collection("DatabaseTests")] -public class PostgresTests -{ - // 50 tests, 15 minutes -} - -[Collection("DatabaseTests")] -public class MySqlTests -{ - // 30 tests, 10 minutes -} - -// Slow container tests - separate group -[Collection("ContainerTests")] -public class DockerTests -{ - // 40 tests, 12 minutes -} - -// Fast tests - no collection (run together) -public class QuickTests -{ - // 100 tests, 2 minutes -} - -public class UnitTests -{ - // 200 tests, 3 minutes -} -``` - -**Result**: 3 parallel jobs -1. **Collection_DatabaseTests**: PostgresTests + MySqlTests (~25 min) -2. **Collection_ContainerTests**: DockerTests (~12 min) -3. **UncollectedTests**: QuickTests + UnitTests (~5 min) - -**Total CI time**: ~25 min (previously 55+ min) - -### Example 3: All Collections (Maximum Splitting) - -```csharp -[Collection("PostgresTests")] -public class PostgresTests { } - -[Collection("MySqlTests")] -public class MySqlTests { } - -[Collection("DockerTests")] -public class DockerTests { } -``` - -**Result**: 3 jobs (one per collection), no uncollected job - -### Example 4: Exclude Certain Collections - -```xml - - true - Aspire.Hosting.Tests - - - QuickTests;FastTests - -``` - -```csharp -[Collection("SlowTests")] -public class SlowTests { } // Gets own job - -[Collection("QuickTests")] -public class QuickTests { } // Runs in UncollectedTests job - -public class OtherTests { } // Runs in UncollectedTests job -``` - -**Result**: 2 jobs -1. **Collection_SlowTests** -2. **UncollectedTests** (QuickTests + OtherTests) - -## Configuration Properties - -### New in v2 - -```xml - -false - - -Collection1;Collection2 - - -20m -10m -``` - -### Per-Collection Timeouts (Advanced) - -```xml - - - 30m - 25m - -``` - -## Decision Tree - -``` -Is the test project slow (>15 minutes)? -│ -├─ NO → Don't enable splitting -│ (Keep as regular test) -│ -└─ YES → Do you have groups of slow tests? - │ - ├─ NO → Don't enable splitting OR use simple splitting - │ (All tests in one job is fine) - │ - └─ YES → Use collection-based splitting! - │ - Step 1: Add [Collection("GroupName")] to slow test groups - Step 2: Set SplitTestsForCI=true - Step 3: Set TestClassNamesPrefix - Step 4: Leave fast tests without collection attribute - │ - Result: N+1 jobs (N collections + 1 uncollected) -``` - -## Migration Strategy - -### Phase 1: Infrastructure (Week 1) -- Implement v2 MSBuild targets with collection discovery -- Update PowerShell script to generate collection-based matrices -- Test with example project (no actual collections yet) - -### Phase 2: Migrate Templates.Tests (Week 2) -- Keep NO collections initially (verify 1 job = current behavior) -- Optionally add collections if beneficial -- Validate backward compatibility - -### Phase 3: Enable Hosting.Tests (Week 3) -- Analyze test suite to identify slow groups -- Add `[Collection]` attributes to slow test groups -- Enable `SplitTestsForCI=true` -- Compare CI times before/after - -### Phase 4: Rollout & Optimize (Week 4) -- Apply to other long-running projects -- Fine-tune collection groupings based on actual times -- Document best practices - -## Best Practices - -### When to Use Collections - -✅ **DO** use collections for: -- Tests that share expensive setup/teardown -- Tests that use the same test fixtures -- Long-running integration tests that can be grouped logically -- Tests that have similar resource requirements - -❌ **DON'T** use collections for: -- Fast unit tests (let them run together in uncollected job) -- Tests that should be isolated -- Creating too many tiny collections (overhead not worth it) - -### Recommended Groupings - -```csharp -// Good: Logical grouping of slow tests -[Collection("DatabaseIntegrationTests")] -public class PostgresIntegrationTests { } - -[Collection("DatabaseIntegrationTests")] -public class SqlServerIntegrationTests { } - -// Good: Resource-specific grouping -[Collection("DockerContainerTests")] -public class ContainerLifecycleTests { } - -[Collection("DockerContainerTests")] -public class ContainerNetworkingTests { } - -// Bad: Too granular (defeats the purpose) -[Collection("PostgresTest1")] -public class PostgresTest1 { } - -[Collection("PostgresTest2")] -public class PostgresTest2 { } -``` - -## Expected Outcomes - -### Before (Monolithic) -``` -Aspire.Hosting.Tests: 1 job, 60 minutes -``` - -### After (Collection-Based Splitting) -``` -Collection_DatabaseTests: 1 job, 25 minutes -Collection_ContainerTests: 1 job, 20 minutes -Collection_AzureTests: 1 job, 15 minutes -UncollectedTests: 1 job, 10 minutes -``` - -**Total CI time**: ~25 minutes (jobs run in parallel) -**Job count**: 4 jobs (manageable) -**Time saved**: 35 minutes (58% reduction) - -## Success Criteria - -- ✅ All OSes generate correct collection-based matrices -- ✅ Collection tests run together in single jobs -- ✅ Uncollected tests run together in one job -- ✅ No tests are accidentally skipped -- ✅ CI time for long-running projects reduced by 50%+ -- ✅ Number of jobs remains manageable (<10 per project per OS) -- ✅ Works on both GitHub Actions and Azure DevOps - -## Rollback Plan - -If issues arise: -1. Set `DisableCollectionBasedSplitting=true` to use v1 class-based splitting -2. Or set `SplitTestsForCI=false` to disable all splitting -3. No YAML changes needed (matrix adapts automatically) - -## Next Steps - -1. Review this updated v2 plan -2. Implement Step 1 (MSBuild targets with collection discovery) -3. Implement Step 2 (PowerShell script with collection matrix generation) -4. Test with sample collections -5. Roll out to Hosting.Tests -6. Monitor and optimize - ---- - -**Key Innovation**: v2 uses xUnit collections to create **logical test groups** while keeping fast tests together, resulting in optimal parallelization with minimal job overhead. \ No newline at end of file diff --git a/docs/test-splitting/IMPLEMENTATION_PLAN_V3.md b/docs/test-splitting/IMPLEMENTATION_PLAN_V3.md deleted file mode 100644 index c22a5662de8..00000000000 --- a/docs/test-splitting/IMPLEMENTATION_PLAN_V3.md +++ /dev/null @@ -1,332 +0,0 @@ -# Test Splitting Implementation Plan v3 - Auto-Detection - -**Date**: 2025-10-16 -**Author**: @radical -**User**: radical -**Objective**: Implement automatic detection of splitting strategy: -- Collections present → Split by collection + uncollected -- No collections → Split by class (original behavior) -- No `SplitTestsOnCI` → No splitting (run as single job) - -## Overview - -This v3 plan simplifies configuration by automatically detecting the appropriate splitting strategy. - -## Auto-Detection Logic - -``` -Is SplitTestsOnCI=true? - │ - ├─ NO → Run as single job (no splitting) - │ - └─ YES → Build project and extract test metadata - │ - ├─ Has Collections? → Split by Collection + Uncollected - │ Result: N+1 jobs (one per collection + one uncollected) - │ - └─ No Collections? → Split by Class - Result: N jobs (one per test class) -``` - -## Splitting Modes - -### Mode 1: No Splitting (Default) - -```xml - - - -``` - -**Result**: 1 job running entire test project - -### Mode 2: Collection-Based Splitting (Auto-Detected) - -```xml - - true - Aspire.Hosting.Tests - -``` - -```csharp -[Collection("DatabaseTests")] -public class PostgresTests { } - -[Collection("ContainerTests")] -public class DockerTests { } - -public class QuickTests { } // No collection -``` - -**Detection**: Collections found → Use collection-based splitting -**Result**: 3 jobs (DatabaseTests, ContainerTests, Uncollected) - -### Mode 3: Class-Based Splitting (Auto-Detected) - -```xml - - true - Aspire.Templates.Tests - -``` - -```csharp -// No [Collection] attributes on any test class -public class Test1 { } -public class Test2 { } -public class Test3 { } -``` - -**Detection**: No collections found → Use class-based splitting -**Result**: 3 jobs (Test1, Test2, Test3) - -## Architecture - -### Phase 1: Discovery (MSBuild) - -``` -ExtractTestClassNames Target - ↓ -Run: dotnet .dll --list-tests - ↓ -Parse output with PowerShell helper - ↓ -Detect collections using regex - ↓ - ├─ Collections found? - │ └─ Write: collection:Name, uncollected:* - │ - └─ No collections? - └─ Write: class:FullClassName (one per class) -``` - -### Phase 2: Matrix Generation (PowerShell) - -``` -generate-test-matrix.ps1 - ↓ -Read .tests.list file - ↓ -Parse entries - ↓ - ├─ Type: collection - │ └─ Generate: Collection jobs + Uncollected job - │ - └─ Type: class - └─ Generate: One job per class -``` - -## Implementation Components - -### 1. PowerShell Discovery Helper - -New script: `eng/scripts/extract-test-metadata.ps1` - -Parses `--list-tests` output to detect collections. - -### 2. Enhanced MSBuild Target - -`ExtractTestClassNames` target calls PowerShell helper to detect mode. - -### 3. Enhanced Matrix Generator - -`generate-test-matrix.ps1` handles both collection and class entries. - -## File Formats - -### .tests.list Format (Auto-Generated) - -**Collection-based mode** (collections detected): -``` -collection:DatabaseTests -collection:ContainerTests -uncollected:* -``` - -**Class-based mode** (no collections): -``` -class:Aspire.Templates.Tests.Test1 -class:Aspire.Templates.Tests.Test2 -class:Aspire.Templates.Tests.Test3 -``` - -### Matrix Output - -**Collection-based**: -```json -{ - "include": [ - { - "type": "collection", - "name": "DatabaseTests", - "filterArg": "--filter-collection \"DatabaseTests\"", - ... - }, - { - "type": "uncollected", - "name": "UncollectedTests", - "filterArg": "--filter-not-collection \"DatabaseTests\" ...", - ... - } - ] -} -``` - -**Class-based**: -```json -{ - "include": [ - { - "type": "class", - "fullClassName": "Aspire.Templates.Tests.Test1", - "filterArg": "--filter-class \"Aspire.Templates.Tests.Test1\"", - ... - }, - { - "type": "class", - "fullClassName": "Aspire.Templates.Tests.Test2", - "filterArg": "--filter-class \"Aspire.Templates.Tests.Test2\"", - ... - } - ] -} -``` - -## Benefits - -1. **Zero Configuration**: Just set `SplitTestsOnCI=true` and it works -2. **Automatic Optimization**: Uses collections if present, falls back to classes -3. **Backward Compatible**: Existing projects work without changes -4. **Developer-Friendly**: Add `[Collection]` when needed, remove when not -5. **Flexible**: Can mix modes across different projects - -## Configuration Properties - -### Minimal Configuration - -```xml - - - true - YourProject.Tests - -``` - -### Optional Overrides - -```xml - - - 25m - 12m - - - 15m - FastTests - - - false - false - false - -``` - -## Implementation Steps - -1. [Step 1: PowerShell Discovery Helper](./STEP_01_DISCOVERY_HELPER.md) -2. [Step 2: MSBuild Targets (v3)](./STEP_02_MSBUILD_TARGETS_V3.md) -3. [Step 3: Matrix Generator (v3)](./STEP_03_MATRIX_GENERATOR_V3.md) -4. [Step 4: GitHub Actions (No Changes)](./STEP_03_GITHUB_ACTIONS.md) -5. [Step 5: Project Configuration (v3)](./STEP_04_PROJECT_CONFIG_V3.md) -6. [Step 6: Testing & Migration](./STEP_05_TESTING_V3.md) - -## Migration Examples - -### Example 1: Aspire.Templates.Tests - -**Current** (custom mechanism): -```xml -true -Aspire.Templates.Tests -``` - -**After v3** (unified, auto-detect): -```xml -true -Aspire.Templates.Tests -``` - -**Auto-detected mode**: Class-based (no collections in templates tests) -**Result**: Same behavior as before (one job per test class) - -### Example 2: Aspire.Hosting.Tests (NEW) - -```xml -true -Aspire.Hosting.Tests -``` - -**Option A**: Leave tests as-is (no collections) -- **Auto-detected mode**: Class-based -- **Result**: One job per test class (~50 jobs) - -**Option B**: Add collections to slow tests -```csharp -[Collection("DatabaseTests")] -public class PostgresTests { } - -[Collection("DatabaseTests")] -public class MySqlTests { } - -public class QuickTests { } // No collection -``` - -- **Auto-detected mode**: Collection-based -- **Result**: 3 jobs (DatabaseTests, Uncollected with QuickTests, etc.) - -## Decision Tree - -``` -Want to split tests? -│ -├─ NO → Don't set SplitTestsOnCI -│ Result: 1 job (current behavior) -│ -└─ YES → Set SplitTestsOnCI=true - │ - Do you have logical test groups? - │ - ├─ YES → Add [Collection] attributes - │ Result: Auto-detected collection mode - │ Jobs: N collections + 1 uncollected - │ - └─ NO → Leave tests as-is - Result: Auto-detected class mode - Jobs: One per class -``` - -## Success Criteria - -- ✅ Auto-detection works for both modes -- ✅ No breaking changes to existing projects -- ✅ Templates.Tests migrates cleanly -- ✅ Hosting.Tests can use either mode -- ✅ All 3 OSes work correctly -- ✅ Clear logging shows which mode was detected -- ✅ CI times reduced by 50%+ for long-running projects - -## Next Steps - -1. Review v3 plan -2. Implement discovery helper script -3. Update MSBuild targets with auto-detection -4. Update matrix generator to handle both modes -5. Test with both collection and class modes -6. Migrate Templates.Tests as proof-of-concept -7. Enable Hosting.Tests with collections -8. Document best practices - ---- - -**Key Innovation**: v3 uses **automatic detection** to choose the optimal splitting strategy, eliminating configuration complexity while maintaining flexibility. \ No newline at end of file diff --git a/docs/test-splitting/IMPLEMENTATION_SUMMARY.md b/docs/test-splitting/IMPLEMENTATION_SUMMARY.md deleted file mode 100644 index 7b796377399..00000000000 --- a/docs/test-splitting/IMPLEMENTATION_SUMMARY.md +++ /dev/null @@ -1,255 +0,0 @@ -# Test Splitting Implementation - Summary & Checklist - -**Date**: 2025-01-16 -**Author**: @radical -**Status**: Ready for Implementation - -## Overview - -This implementation adds automatic test splitting to dotnet/aspire CI, reducing test execution time by running tests in parallel. - -**Key Innovation**: Auto-detection of splitting strategy -- Has `[Collection]` attributes? → Split by collection + uncollected -- No collections? → Split by test class -- Not enabled? → Run as single job (no change) - -## What's Being Implemented - -### New Files - -1. **`eng/scripts/extract-test-metadata.ps1`** (Step 1) - - Parses `--list-tests` output - - Detects collections vs classes - - Outputs `.tests.list` file - -2. **`eng/scripts/generate-test-matrix.ps1`** (Step 3) - - Reads `.tests.list` and `.tests.metadata.json` - - Generates JSON matrix for CI - - Handles both collection and class modes - -### Modified Files - -3. **`tests/Directory.Build.targets`** (Step 2) - - Enhanced `ExtractTestClassNames` target - - Calls PowerShell discovery helper - - Writes metadata for matrix generation - -4. **`tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj`** (Step 4) - - Migrate from old custom mechanism - - Use new unified `SplitTestsOnCI` property - -### Existing Files (No Changes) - -- `.github/workflows/tests.yml` - Already supports new matrix format -- `.github/actions/enumerate-tests/action.yml` - Already calls scripts correctly -- `tests/Shared/GetTestProjects.proj` - Already orchestrates correctly - -## Implementation Checklist - -### Phase 1: Infrastructure (Week 1) - -- [ ] **Create `eng/scripts/extract-test-metadata.ps1`** - - [ ] Copy from STEP_01_DISCOVERY_HELPER.md - - [ ] Test with mock data (see Step 5) - - [ ] Verify collections detected correctly - - [ ] Verify class-only mode works - -- [ ] **Create `eng/scripts/generate-test-matrix.ps1`** - - [ ] Copy from STEP_03_MATRIX_GENERATOR_V3.md - - [ ] Test with sample .tests.list files (see Step 5) - - [ ] Verify JSON output is valid - - [ ] Test both collection and class modes - -- [ ] **Update `tests/Directory.Build.targets`** - - [ ] Add enhanced ExtractTestClassNames target from STEP_02_MSBUILD_TARGETS_V3.md - - [ ] Test locally with `dotnet build` (see Step 5) - - [ ] Verify `.tests.list` and `.tests.metadata.json` are created - - [ ] Check binlog for errors - -### Phase 2: Migrate Templates.Tests (Week 2) - -- [ ] **Update `tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj`** - - [ ] Replace `ExtractTestClassNamesForHelix` with `SplitTestsOnCI` - - [ ] Add `RequiresNugetsForSplitTests=true` - - [ ] Add `RequiresTestSdkForSplitTests=true` - - [ ] Add `EnablePlaywrightInstallForSplitTests=true` - - [ ] Remove `TestArchiveTestsDir` override - -- [ ] **Test Locally** - - [ ] Build project with splitting enabled - - [ ] Verify class-based mode detected (no collections in templates tests) - - [ ] Check `.tests.list` has `class:` entries - - [ ] Verify matrix has same number of jobs as before - -- [ ] **Create PR** - - [ ] Title: "Migrate Aspire.Templates.Tests to unified test splitting" - - [ ] Link to this implementation plan - - [ ] Test in CI - - [ ] Verify same behavior as before - -### Phase 3: Enable Hosting.Tests (Week 3) - -- [ ] **Add Collections to Slow Tests** - - [ ] Identify slow test groups (>10 min combined) - - [ ] Add `[Collection("DatabaseTests")]` to database test classes - - [ ] Add `[Collection("ContainerTests")]` to container test classes - - [ ] Leave fast tests without `[Collection]` attribute - -- [ ] **Update `tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj`** - - [ ] Add `SplitTestsOnCI=true` - - [ ] Add `TestClassNamesPrefix=Aspire.Hosting.Tests` - - [ ] Set timeouts (see Step 4) - -- [ ] **Test Locally** - - [ ] Build with splitting enabled - - [ ] Verify collection-based mode detected - - [ ] Check `.tests.list` has `collection:` entries - - [ ] Test filters work (see Step 5) - -- [ ] **Create PR** - - [ ] Title: "Enable test splitting for Aspire.Hosting.Tests" - - [ ] Document expected CI time improvement - - [ ] Monitor CI times after merge - -### Phase 4: Rollout & Optimize (Week 4) - -- [ ] **Identify Other Long-Running Projects** - - [ ] Review CI times for all test projects - - [ ] List projects > 15 minutes - - [ ] Prioritize by impact - -- [ ] **Enable Splitting Incrementally** - - [ ] One project per PR - - [ ] Monitor each for issues - - [ ] Adjust collection groupings as needed - -- [ ] **Document Best Practices** - - [ ] Collection size guidelines - - [ ] When to split vs not split - - [ ] Troubleshooting common issues - -## Testing Strategy - -### Local Testing (Before Each PR) - -1. **Unit Test Scripts** - - [ ] Test `extract-test-metadata.ps1` with mock data - - [ ] Test `generate-test-matrix.ps1` with sample files - - [ ] Verify JSON output structure - -2. **Integration Test MSBuild** - - [ ] Build test project with splitting enabled - - [ ] Verify files generated in `artifacts/helix/` - - [ ] Check mode detection is correct - -3. **End-to-End Test** - - [ ] Run full `GetTestProjects.proj` - - [ ] Generate matrix JSON - - [ ] Validate matrix structure - - [ ] Test xUnit filters work - -### CI Testing (After Push) - -1. **Setup Jobs** - - [ ] All 3 OS setup jobs succeed - - [ ] Matrices are generated - - [ ] Artifacts are uploaded - -2. **Split Test Jobs** - - [ ] New jobs appear as expected - - [ ] Tests run with correct filters - - [ ] Results are uploaded - - [ ] No unexpected failures - -3. **Performance** - - [ ] CI times reduced as expected - - [ ] No increase in flakiness - - [ ] Resource usage acceptable - -## Success Criteria - -### Functional - -- [ ] Auto-detection works (collection vs class mode) -- [ ] Templates.Tests migrates without behavior change -- [ ] Hosting.Tests splits into ~3-5 jobs -- [ ] All tests pass in split jobs -- [ ] Test results are properly reported -- [ ] Works on all 3 OSes (Linux, macOS, Windows) - -### Performance - -- [ ] Hosting.Tests CI time reduced by 50%+ -- [ ] No increase in test flakiness -- [ ] Job count remains manageable (<10 per project per OS) - -### Maintainability - -- [ ] Clear documentation for developers -- [ ] Easy to enable for new projects -- [ ] Easy to troubleshoot issues -- [ ] No breaking changes to existing projects - -## Rollback Plan - -If critical issues arise: - -### Per-Project Rollback - -```xml - - -``` - -Project reverts to single-job execution immediately. - -### Full Rollback - -Revert the PR that modified `Directory.Build.targets`. -All projects revert to original behavior. - -## File Reference - -| Step | File(s) | Purpose | -|------|---------|---------| -| 1 | `STEP_01_DISCOVERY_HELPER.md` | PowerShell script to detect collections/classes | -| 2 | `STEP_02_MSBUILD_TARGETS_V3.md` | MSBuild target that calls discovery helper | -| 3 | `STEP_03_MATRIX_GENERATOR_V3.md` | PowerShell script to generate JSON matrices | -| 4 | `STEP_04_PROJECT_CONFIG_V3.md` | How to configure test projects | -| 5 | `STEP_05_TESTING_V3.md` | Local testing guide | -| 6 | `STEP_06_CI_INTEGRATION.md` | CI verification guide | - -## Questions for Copilot - -Before starting implementation, Copilot should clarify: - -1. **Templates.Tests Migration**: Should we remove the old `enumerate-tests` template-specific logic in the workflow, or keep it as fallback? - -2. **Timeout Defaults**: What should default timeout values be if not specified? - - Suggested: `SplitTestSessionTimeout=20m`, `UncollectedTestsSessionTimeout=15m` - -3. **Collection Naming**: Any conventions or restrictions on collection names? - - Suggested: Alphanumeric + underscore only - -4. **Error Handling**: Should we fail CI if splitting is enabled but no tests found, or fall back to running all tests? - - Suggested: Fail fast to catch configuration errors early - -5. **Artifacts**: Should we always upload `.tests.list` and `.tests.metadata.json` files, even on success? - - Suggested: Yes, for debugging and transparency - -## Ready for Implementation? - -- [x] All design documents complete -- [x] Testing strategy defined -- [x] Success criteria clear -- [x] Rollback plan in place -- [x] Questions for Copilot identified - -**Status**: ✅ Ready to hand off to Copilot for PR creation - -**Estimated Implementation Time**: 2-3 hours for infrastructure + testing - -**Recommended Approach**: Implement in 3 separate PRs: -1. PR #1: Add infrastructure (scripts + targets) - test with Templates.Tests -2. PR #2: Enable Hosting.Tests with collections -3. PR #3: Roll out to remaining long-running projects \ No newline at end of file diff --git a/docs/test-splitting/STEP_01_DISCOVERY_HELPER.md b/docs/test-splitting/STEP_01_DISCOVERY_HELPER.md deleted file mode 100644 index 65b645eb1b9..00000000000 --- a/docs/test-splitting/STEP_01_DISCOVERY_HELPER.md +++ /dev/null @@ -1,368 +0,0 @@ -# Step 1: PowerShell Discovery Helper - -## Overview - -Create a PowerShell helper script that parses `--list-tests` output to detect xUnit collections and test classes, determining the optimal splitting mode. - -## File: `eng/scripts/extract-test-metadata.ps1` - -### Complete Implementation - -```powershell -<# -.SYNOPSIS - Extracts test metadata (collections or classes) from xUnit test assembly. - -.DESCRIPTION - Parses output of 'dotnet test.dll --list-tests' to determine: - - Are collections present? → Use collection-based splitting - - No collections? → Use class-based splitting - - Outputs a structured list file for consumption by matrix generation. - -.PARAMETER TestAssemblyOutput - The console output from running the test assembly with --list-tests - -.PARAMETER TestClassNamesPrefix - Prefix to filter test classes (e.g., "Aspire.Hosting.Tests") - -.PARAMETER TestCollectionsToSkip - Semicolon-separated list of collection names to exclude from splitting - -.PARAMETER OutputListFile - Path to write the .tests.list file - -.EXAMPLE - $output = & dotnet MyTests.dll --list-tests - .\extract-test-metadata.ps1 -TestAssemblyOutput $output -TestClassNamesPrefix "MyTests" -OutputListFile "./tests.list" - -.NOTES - Author: Aspire Team (@radical) - Date: 2025-10-16 - Version: 3.0 - Requires: PowerShell 7.0+ -#> - -[CmdletBinding()] -param( - [Parameter(Mandatory=$true, HelpMessage="Output from test assembly --list-tests")] - [string[]]$TestAssemblyOutput, - - [Parameter(Mandatory=$true, HelpMessage="Prefix for test class names")] - [string]$TestClassNamesPrefix, - - [Parameter(Mandatory=$false, HelpMessage="Collections to skip (semicolon-separated)")] - [string]$TestCollectionsToSkip = "", - - [Parameter(Mandatory=$true, HelpMessage="Output file path")] - [string]$OutputListFile -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version Latest - -#region Helper Functions - -function Write-Message { - param( - [string]$Message, - [ValidateSet('Info', 'Success', 'Warning', 'Error', 'Debug')] - [string]$Level = 'Info' - ) - - $prefix = switch ($Level) { - 'Success' { '✅' } - 'Warning' { '⚠️' } - 'Error' { '❌' } - 'Debug' { '🔍' } - default { 'ℹ️' } - } - - Write-Host "$prefix $Message" -} - -#endregion - -#region Parse Test Output - -Write-Message "Parsing test assembly output..." -Level Info - -# xUnit v3 output format when listing tests: -# The test assembly output includes test names with their collection information. -# We need to extract both collections and class names. - -$collections = [System.Collections.Generic.HashSet[string]]::new() -$testClasses = [System.Collections.Generic.HashSet[string]]::new() - -# Regex patterns -$testNameRegex = "^\s*($TestClassNamesPrefix[^\(]+)" -$collectionIndicator = "Collection:" # xUnit prints this before test names in a collection - -$currentCollection = $null - -foreach ($line in $TestAssemblyOutput) { - # Check if this line indicates a collection - if ($line -match "^\s*$collectionIndicator\s*(.+)$") { - $currentCollection = $Matches[1].Trim() - Write-Message " Found collection: $currentCollection" -Level Debug - [void]$collections.Add($currentCollection) - continue - } - - # Check if this is a test name line - if ($line -match $testNameRegex) { - $fullTestName = $Matches[1].Trim() - - # Extract class name from test name - # Format: "Namespace.ClassName.MethodName" - if ($fullTestName -match "^($TestClassNamesPrefix\.[^\.]+)\.") { - $className = $Matches[1] - [void]$testClasses.Add($className) - } - } -} - -#endregion - -#region Filter Collections - -$collectionsToSkipList = if ($TestCollectionsToSkip) { - $TestCollectionsToSkip -split ';' | ForEach-Object { $_.Trim() } | Where-Object { $_ } -} else { - @() -} - -$filteredCollections = $collections | Where-Object { $_ -notin $collectionsToSkipList } - -#endregion - -#region Determine Splitting Mode - -$hasCollections = $filteredCollections.Count -gt 0 -$mode = if ($hasCollections) { "collection" } else { "class" } - -Write-Message "" -Level Info -Write-Message "Detection Results:" -Level Success -Write-Message " Mode: $mode" -Level Info -Write-Message " Collections found: $($collections.Count)" -Level Info -Write-Message " Collections after filtering: $($filteredCollections.Count)" -Level Info -Write-Message " Test classes found: $($testClasses.Count)" -Level Info - -if ($collectionsToSkipList.Count -gt 0) { - Write-Message " Skipped collections: $($collectionsToSkipList -join ', ')" -Level Info -} - -#endregion - -#region Generate Output File - -$outputLines = [System.Collections.Generic.List[string]]::new() - -if ($mode -eq "collection") { - Write-Message "" -Level Info - Write-Message "Using COLLECTION-BASED splitting" -Level Success - - # Add collection entries - foreach ($collection in ($filteredCollections | Sort-Object)) { - $outputLines.Add("collection:$collection") - Write-Message " + Job: Collection_$collection" -Level Debug - } - - # Always add uncollected entry - $outputLines.Add("uncollected:*") - Write-Message " + Job: Uncollected (tests without collections)" -Level Debug - - Write-Message "" -Level Info - Write-Message "Expected jobs: $($filteredCollections.Count + 1) ($($filteredCollections.Count) collections + 1 uncollected)" -Level Success -} -else { - Write-Message "" -Level Info - Write-Message "Using CLASS-BASED splitting" -Level Success - - # Add class entries - foreach ($className in ($testClasses | Sort-Object)) { - $outputLines.Add("class:$className") - $shortName = $className -replace "^$TestClassNamesPrefix\.", "" - Write-Message " + Job: $shortName" -Level Debug - } - - Write-Message "" -Level Info - Write-Message "Expected jobs: $($testClasses.Count) (one per class)" -Level Success -} - -#endregion - -#region Write Output File - -# Ensure output directory exists -$outputDir = [System.IO.Path]::GetDirectoryName($OutputListFile) -if ($outputDir -and -not (Test-Path $outputDir)) { - New-Item -ItemType Directory -Path $outputDir -Force | Out-Null -} - -# Write file -$outputLines | Set-Content -Path $OutputListFile -Encoding UTF8 - -Write-Message "" -Level Info -Write-Message "Output written to: $OutputListFile" -Level Success -Write-Message "Lines: $($outputLines.Count)" -Level Info - -#endregion -``` - -## Usage Examples - -### Example 1: Project with Collections - -```powershell -# Run test assembly -$output = & dotnet artifacts/bin/Aspire.Hosting.Tests/Debug/net9.0/Aspire.Hosting.Tests.dll --list-tests - -# Extract metadata -.\eng\scripts\extract-test-metadata.ps1 ` - -TestAssemblyOutput $output ` - -TestClassNamesPrefix "Aspire.Hosting.Tests" ` - -OutputListFile "./artifacts/helix/Aspire.Hosting.Tests.tests.list" -``` - -**Console Output**: -``` -ℹ️ Parsing test assembly output... -🔍 Found collection: DatabaseTests -🔍 Found collection: ContainerTests - -✅ Detection Results: -ℹ️ Mode: collection -ℹ️ Collections found: 2 -ℹ️ Collections after filtering: 2 -ℹ️ Test classes found: 15 - -✅ Using COLLECTION-BASED splitting -🔍 + Job: Collection_DatabaseTests -🔍 + Job: Collection_ContainerTests -🔍 + Job: Uncollected (tests without collections) - -✅ Expected jobs: 3 (2 collections + 1 uncollected) - -✅ Output written to: ./artifacts/helix/Aspire.Hosting.Tests.tests.list -ℹ️ Lines: 3 -``` - -**Output File** (`Aspire.Hosting.Tests.tests.list`): -``` -collection:ContainerTests -collection:DatabaseTests -uncollected:* -``` - -### Example 2: Project without Collections - -```powershell -$output = & dotnet artifacts/bin/Aspire.Templates.Tests/Debug/net9.0/Aspire.Templates.Tests.dll --list-tests - -.\eng\scripts\extract-test-metadata.ps1 ` - -TestAssemblyOutput $output ` - -TestClassNamesPrefix "Aspire.Templates.Tests" ` - -OutputListFile "./artifacts/helix/Aspire.Templates.Tests.tests.list" -``` - -**Console Output**: -``` -ℹ️ Parsing test assembly output... - -✅ Detection Results: -ℹ️ Mode: class -ℹ️ Collections found: 0 -ℹ️ Collections after filtering: 0 -ℹ️ Test classes found: 12 - -✅ Using CLASS-BASED splitting -🔍 + Job: BuildAndRunStarterTemplateBuiltInTest -🔍 + Job: BuildAndRunTemplateTests -🔍 + Job: EmptyTemplateRunTests -... - -✅ Expected jobs: 12 (one per class) - -✅ Output written to: ./artifacts/helix/Aspire.Templates.Tests.tests.list -ℹ️ Lines: 12 -``` - -**Output File** (`Aspire.Templates.Tests.tests.list`): -``` -class:Aspire.Templates.Tests.BuildAndRunStarterTemplateBuiltInTest -class:Aspire.Templates.Tests.BuildAndRunTemplateTests -class:Aspire.Templates.Tests.EmptyTemplateRunTests -class:Aspire.Templates.Tests.MSTest_PerTestFrameworkTemplatesTests -class:Aspire.Templates.Tests.NewUpAndBuildStandaloneTemplateTests -class:Aspire.Templates.Tests.None_StarterTemplateProjectNamesTests -class:Aspire.Templates.Tests.Nunit_PerTestFrameworkTemplatesTests -class:Aspire.Templates.Tests.Nunit_StarterTemplateProjectNamesTests -class:Aspire.Templates.Tests.StarterTemplateRunTests -class:Aspire.Templates.Tests.StarterTemplateWithTestsRunTests -class:Aspire.Templates.Tests.Xunit_PerTestFrameworkTemplatesTests -class:Aspire.Templates.Tests.Xunit_StarterTemplateProjectNamesTests -``` - -### Example 3: Skip Certain Collections - -```powershell -.\eng\scripts\extract-test-metadata.ps1 ` - -TestAssemblyOutput $output ` - -TestClassNamesPrefix "Aspire.Hosting.Tests" ` - -TestCollectionsToSkip "QuickTests;FastTests" ` - -OutputListFile "./artifacts/helix/Aspire.Hosting.Tests.tests.list" -``` - -**Result**: QuickTests and FastTests won't get their own jobs; they'll run in the uncollected job. - -## Testing the Script - -### Test 1: Mock Collection Output - -```powershell -$mockOutput = @( - "Collection: DatabaseTests", - " Aspire.Hosting.Tests.PostgresTests.CanStartContainer", - " Aspire.Hosting.Tests.PostgresTests.CanConnectToDatabase", - "Collection: ContainerTests", - " Aspire.Hosting.Tests.DockerTests.CanStartGenericContainer", - "Aspire.Hosting.Tests.QuickTests.FastTest1", - "Aspire.Hosting.Tests.QuickTests.FastTest2" -) - -.\eng\scripts\extract-test-metadata.ps1 ` - -TestAssemblyOutput $mockOutput ` - -TestClassNamesPrefix "Aspire.Hosting.Tests" ` - -OutputListFile "./test-output.list" -``` - -**Expected**: -- Mode: collection -- Collections: DatabaseTests, ContainerTests -- Output: 3 lines (2 collections + uncollected) - -### Test 2: Mock Class-Only Output - -```powershell -$mockOutput = @( - "Aspire.Templates.Tests.Test1.Method1", - "Aspire.Templates.Tests.Test1.Method2", - "Aspire.Templates.Tests.Test2.Method1", - "Aspire.Templates.Tests.Test3.Method1" -) - -.\eng\scripts\extract-test-metadata.ps1 ` - -TestAssemblyOutput $mockOutput ` - -TestClassNamesPrefix "Aspire.Templates.Tests" ` - -OutputListFile "./test-output.list" -``` - -**Expected**: -- Mode: class -- Classes: Test1, Test2, Test3 -- Output: 3 lines (one per class) - -## Next Steps - -Proceed to [Step 2: MSBuild Targets (v3)](./STEP_02_MSBUILD_TARGETS_V3.md) \ No newline at end of file diff --git a/docs/test-splitting/STEP_01_MSBUILD_TARGETS.md b/docs/test-splitting/STEP_01_MSBUILD_TARGETS.md deleted file mode 100644 index 8ceb86d41b1..00000000000 --- a/docs/test-splitting/STEP_01_MSBUILD_TARGETS.md +++ /dev/null @@ -1,236 +0,0 @@ -# Step 1: MSBuild Targets Implementation - -## Overview - -Modify MSBuild targets to support unified test splitting mechanism while maintaining all 3 OS compatibility. - -## File: `tests/Directory.Build.targets` - -### Changes Required - -1. **Add new ExtractTestClassNames target** (replacing existing) -2. **Add metadata generation** -3. **Update GetRunTestsOnGithubActions target** - -### Implementation - -```xml - - - - - - - - - - - - - - - - - - - - - - <_Regex>^\s*($(TestClassNamesPrefix)[^\($]+) - - - - - <_TestLines0 Include="$([System.Text.RegularExpressions.Regex]::Match('%(_ListOfTestsLines.Identity)', '$(_Regex)'))" /> - - - - - - - - - - - - - - - - - <_RelativeProjectPath>$(MSBuildProjectDirectory.Replace('$(RepoRoot)', '')) - <_RelativeProjectPath>$(_RelativeProjectPath.Replace('\', '/')) - - - - <_MetadataLines Include="{" /> - <_MetadataLines Include=" "projectName": "$(MSBuildProjectName)"," /> - <_MetadataLines Include=" "testClassNamesPrefix": "$(TestClassNamesPrefix)"," /> - <_MetadataLines Include=" "testProjectPath": "$(_RelativeProjectPath)/$(MSBuildProjectFile)"," /> - <_MetadataLines Include=" "requiresNugets": "$(RequiresNugetsForSplitTests)"," /> - <_MetadataLines Include=" "requiresTestSdk": "$(RequiresTestSdkForSplitTests)"," /> - <_MetadataLines Include=" "testSessionTimeout": "$(SplitTestSessionTimeout)"," /> - <_MetadataLines Include=" "testHangTimeout": "$(SplitTestHangTimeout)"," /> - <_MetadataLines Include=" "enablePlaywrightInstall": "$(EnablePlaywrightInstallForSplitTests)"" /> - <_MetadataLines Include="}" /> - - - - - - - - - - - - - - - - - - - - -``` - -## File: `tests/Shared/GetTestProjects.proj` - -### Complete Replacement - -```xml - - - - - $(MSBuildThisFileDirectory)..\..\ - $(ArtifactsDir)test-matrices\ - - - - - - - - - - - - <_TestProjectsToExclude Include="$(RepoRoot)tests\Shared\**\*Tests.csproj" /> - <_TestProjectsToExclude Include="$(RepoRoot)tests\testproject\**\*Tests.csproj" /> - <_TestProjectsToExclude Include="$(RepoRoot)tests\TestingAppHost1\**\*Tests.csproj" /> - - - <_TestProjectsToExclude Include="$(RepoRoot)tests\Aspire.EndToEnd.Tests\**\*Tests.csproj" /> - - <_TestProjects Include="$(RepoRoot)tests\**\*Tests.csproj" - Exclude="@(_TestProjectsToExclude)" /> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <_GenerateMatrixScript>$(RepoRoot)eng\scripts\generate-test-matrix.ps1 - <_TestListsDir>$(ArtifactsDir)helix\ - - - - - - - - -``` - -## Testing the MSBuild Changes - -### Local Testing - -```bash -# On Linux/macOS -./build.sh -restore -build -projects tests/Shared/GetTestProjects.proj /p:TestsListOutputPath=$PWD/artifacts/test-list.txt /p:ContinuousIntegrationBuild=true - -# On Windows -.\build.cmd -restore -build -projects tests/Shared/GetTestProjects.proj /p:TestsListOutputPath=%CD%\artifacts\test-list.txt /p:ContinuousIntegrationBuild=true -``` - -### Verify Outputs - -Check these files were created: -- `artifacts/TestsForGithubActions.list` - Regular tests -- `artifacts/TestsForGithubActions.list.split-projects` - Projects to split (if any) - -### Common Issues - -1. **Path separators**: Ensure paths use `/` in JSON output -2. **Empty lists**: If no split projects, `.split-projects` file won't exist (this is OK) -3. **BuildOs detection**: Make sure `BuildOs` property is set correctly - -## Next Steps - -Proceed to [Step 2: PowerShell Script](./STEP_02_POWERSHELL_SCRIPT.md) \ No newline at end of file diff --git a/docs/test-splitting/STEP_01_MSBUILD_TARGETS_V2.md b/docs/test-splitting/STEP_01_MSBUILD_TARGETS_V2.md deleted file mode 100644 index 23f0637edf9..00000000000 --- a/docs/test-splitting/STEP_01_MSBUILD_TARGETS_V2.md +++ /dev/null @@ -1,295 +0,0 @@ -# Step 1: MSBuild Targets Implementation (v2 - Collection Support) - -## Overview - -Enhanced MSBuild targets that discover xUnit collections and generate a hybrid matrix with: -- One job per collection -- One job for all uncollected tests - -## File: `tests/Directory.Build.targets` - -### Complete Enhanced Target - -```xml - - - - - - - - - - - - $(RepoRoot)tests\helix\xunit.runner.json - $(RepositoryEngineeringDir)testing\xunit.runner.json - - - $(TestingPlatformCommandLineArguments) --filter-method $(TestMethod) - $(TestingPlatformCommandLineArguments) --filter-class $(TestClass) - $(TestingPlatformCommandLineArguments) --filter-namespace $(TestNamespace) - - true - false - - - false - - - - - - - - - - - $(OutDir) - - -$(TargetFramework) - - - - - - - - - - - - - - - - - - - - <_CollectionRegex>^\s*Collection:\s*(.+)$ - - - <_ClassRegex>^\s*($(TestClassNamesPrefix)[^\($]+) - - - - - <_CollectionLines Include="$([System.Text.RegularExpressions.Regex]::Match('%(_ListOfTestsLinesWithTraits.Identity)', '$(_CollectionRegex)'))" /> - <_CollectionNames Include="$([System.Text.RegularExpressions.Regex]::Match('%(_CollectionLines.Identity)', '$(_CollectionRegex)').Groups[1].Value)" - Condition="'$([System.Text.RegularExpressions.Regex]::Match('%(_CollectionLines.Identity)', '$(_CollectionRegex)').Success)' == 'true'" /> - - - - - - - - - <_HasCollections>false - <_HasCollections Condition="'@(UniqueCollections->Count())' != '0'">true - - - - - - <_TestListLines Include="collection:%(UniqueCollections.Identity)" /> - - - <_TestListLines Include="uncollected:*" /> - - - - - - - - <_RelativeProjectPath>$(MSBuildProjectDirectory.Replace('$(RepoRoot)', '')) - <_RelativeProjectPath>$(_RelativeProjectPath.Replace('\', '/')) - - - <_CollectionsList>@(UniqueCollections, ';') - - - - <_MetadataLines Include="{" /> - <_MetadataLines Include=" "projectName": "$(MSBuildProjectName)"," /> - <_MetadataLines Include=" "testClassNamesPrefix": "$(TestClassNamesPrefix)"," /> - <_MetadataLines Include=" "testProjectPath": "$(_RelativeProjectPath)/$(MSBuildProjectFile)"," /> - <_MetadataLines Include=" "collections": "$(_CollectionsList)"," /> - <_MetadataLines Include=" "requiresNugets": "$(RequiresNugetsForSplitTests)"," /> - <_MetadataLines Include=" "requiresTestSdk": "$(RequiresTestSdkForSplitTests)"," /> - <_MetadataLines Include=" "testSessionTimeout": "$(SplitTestSessionTimeout)"," /> - <_MetadataLines Include=" "testHangTimeout": "$(SplitTestHangTimeout)"," /> - <_MetadataLines Include=" "uncollectedTestsSessionTimeout": "$(UncollectedTestsSessionTimeout)"," /> - <_MetadataLines Include=" "uncollectedTestsHangTimeout": "$(UncollectedTestsHangTimeout)"," /> - <_MetadataLines Include=" "enablePlaywrightInstall": "$(EnablePlaywrightInstallForSplitTests)"" /> - <_MetadataLines Include="}" /> - - - - - - - - - - - - - - - - - - - - - -``` - -## Key Changes from v1 - -### 1. Collection Discovery - -```xml - - - - -``` - -### 2. Collection Extraction - -```xml - - - - -``` - -### 3. Simplified Test List Format - -```xml - - - <_TestListLines Include="collection:%(UniqueCollections.Identity)" /> - <_TestListLines Include="uncollected:*" /> - -``` - -### 4. Collection Metadata - -```xml - -<_MetadataLines Include=" "collections": "$(_CollectionsList)"," /> -``` - -## Testing the MSBuild Changes - -### Test 1: Project with No Collections - -```bash -# Create a test project without collections -dotnet build tests/SomeProject.Tests/SomeProject.Tests.csproj \ - /t:Build;ExtractTestClassNames \ - -p:PrepareForHelix=true \ - -p:SplitTestsForCI=true \ - -p:TestClassNamesPrefix=SomeProject.Tests -``` - -**Expected `.tests.list` output**: -``` -uncollected:* -``` - -**Expected matrix**: 1 job (UncollectedTests) - -### Test 2: Project with Collections - -Add collections to test classes: - -```csharp -[Collection("DatabaseTests")] -public class PostgresTests { } - -[Collection("DatabaseTests")] -public class MySqlTests { } - -[Collection("ContainerTests")] -public class DockerTests { } - -public class QuickTests { } // No collection -``` - -Build: -```bash -dotnet build tests/SomeProject.Tests/SomeProject.Tests.csproj \ - /t:Build;ExtractTestClassNames \ - -p:PrepareForHelix=true \ - -p:SplitTestsForCI=true \ - -p:TestClassNamesPrefix=SomeProject.Tests -``` - -**Expected `.tests.list` output**: -``` -collection:DatabaseTests -collection:ContainerTests -uncollected:* -``` - -**Expected matrix**: 3 jobs -1. Collection_DatabaseTests -2. Collection_ContainerTests -3. UncollectedTests - -### Test 3: Exclude Collections - -```bash -dotnet build tests/SomeProject.Tests/SomeProject.Tests.csproj \ - /t:Build;ExtractTestClassNames \ - -p:PrepareForHelix=true \ - -p:SplitTestsForCI=true \ - -p:TestClassNamesPrefix=SomeProject.Tests \ - -p:TestCollectionsToSkipSplitting=DatabaseTests -``` - -**Expected `.tests.list` output**: -``` -collection:ContainerTests -uncollected:* -``` - -**Expected matrix**: 2 jobs -1. Collection_ContainerTests -2. UncollectedTests (includes DatabaseTests now) - -## File: `tests/Shared/GetTestProjects.proj` - -No changes needed from v1 - this file just orchestrates the builds and calls the PowerShell script. - -## Next Steps - -Proceed to [Step 2: PowerShell Script (v2)](./STEP_02_POWERSHELL_SCRIPT_V2.md) \ No newline at end of file diff --git a/docs/test-splitting/STEP_02_MSBUILD_TARGETS_V3.md b/docs/test-splitting/STEP_02_MSBUILD_TARGETS_V3.md deleted file mode 100644 index 3b2fc761ac6..00000000000 --- a/docs/test-splitting/STEP_02_MSBUILD_TARGETS_V3.md +++ /dev/null @@ -1,446 +0,0 @@ -# Step 2: MSBuild Targets Implementation (v3 - Auto-Detection) - -## Overview - -Enhanced MSBuild targets that use the PowerShell discovery helper to automatically detect whether to use collection-based or class-based splitting. - -## File: `tests/Directory.Build.targets` - -### Complete Enhanced Implementation - -```xml - - - - - - - - - $(RepoRoot)tests\helix\xunit.runner.json - $(RepositoryEngineeringDir)testing\xunit.runner.json - - - $(TestingPlatformCommandLineArguments) --filter-method $(TestMethod) - $(TestingPlatformCommandLineArguments) --filter-class $(TestClass) - $(TestingPlatformCommandLineArguments) --filter-namespace $(TestNamespace) - - true - - false - - - - - - - - - - - $(OutDir) - - -$(TargetFramework) - - - - - - - - - - - - - - - - - - - - - <_DiscoveryScriptPath>$(RepoRoot)eng\scripts\extract-test-metadata.ps1 - - - <_TestListFile>$(TestArchiveTestsDir)$(MSBuildProjectName).tests.list - <_MetadataFile>$(TestArchiveTestsDir)$(MSBuildProjectName).tests.metadata.json - - - <_RelativeProjectPath>$(MSBuildProjectDirectory.Replace('$(RepoRoot)', '')) - <_RelativeProjectPath>$(_RelativeProjectPath.Replace('\', '/')) - - - <_CollectionsToSkip Condition="'$(TestCollectionsToSkipSplitting)' != ''">$(TestCollectionsToSkipSplitting) - <_CollectionsToSkip Condition="'$(TestCollectionsToSkipSplitting)' == ''"> - - - - - - - - <_TempOutputFile>$(TestArchiveTestsDir)$(MSBuildProjectName).tests.output.tmp - - - - - - - - - <_DiscoveryCommand>pwsh -NoProfile -ExecutionPolicy Bypass -File "$(_DiscoveryScriptPath)" - <_DiscoveryCommand>$(_DiscoveryCommand) -TestAssemblyOutput (Get-Content '$(_TempOutputFile)') - <_DiscoveryCommand>$(_DiscoveryCommand) -TestClassNamesPrefix "$(TestClassNamesPrefix)" - <_DiscoveryCommand Condition="'$(_CollectionsToSkip)' != ''">$(_DiscoveryCommand) -TestCollectionsToSkip "$(_CollectionsToSkip)" - <_DiscoveryCommand>$(_DiscoveryCommand) -OutputListFile "$(_TestListFile)" - - - - - - - - - - - - - - - - - - <_FirstLine>@(_GeneratedListLines->WithMetadataValue('Identity', '@(_GeneratedListLines, 0)')) - <_DetectedMode Condition="$(_FirstLine.StartsWith('collection:'))">collection - <_DetectedMode Condition="$(_FirstLine.StartsWith('class:'))">class - - - <_EntryCount>@(_GeneratedListLines->Count()) - - - - - - - - <_CollectionsList> - - - - <_CollectionLines Include="@(_GeneratedListLines)" Condition="$([System.String]::Copy('%(Identity)').StartsWith('collection:'))" /> - <_CollectionNames Include="$([System.String]::Copy('%(_CollectionLines.Identity)').Substring(11))" /> - - - - <_CollectionsList>@(_CollectionNames, ';') - - - - - <_MetadataLines Include="{" /> - <_MetadataLines Include=" "projectName": "$(MSBuildProjectName)"," /> - <_MetadataLines Include=" "testClassNamesPrefix": "$(TestClassNamesPrefix)"," /> - <_MetadataLines Include=" "testProjectPath": "$(_RelativeProjectPath)/$(MSBuildProjectFile)"," /> - <_MetadataLines Include=" "mode": "$(_DetectedMode)"," /> - <_MetadataLines Include=" "collections": "$(_CollectionsList)"," /> - <_MetadataLines Include=" "requiresNugets": "$(RequiresNugetsForSplitTests)"," /> - <_MetadataLines Include=" "requiresTestSdk": "$(RequiresTestSdkForSplitTests)"," /> - <_MetadataLines Include=" "testSessionTimeout": "$(SplitTestSessionTimeout)"," /> - <_MetadataLines Include=" "testHangTimeout": "$(SplitTestHangTimeout)"," /> - <_MetadataLines Include=" "uncollectedTestsSessionTimeout": "$(UncollectedTestsSessionTimeout)"," /> - <_MetadataLines Include=" "uncollectedTestsHangTimeout": "$(UncollectedTestsHangTimeout)"," /> - <_MetadataLines Include=" "enablePlaywrightInstall": "$(EnablePlaywrightInstallForSplitTests)"" /> - <_MetadataLines Include="}" /> - - - - - - - - - - - - - - - - - - - - - - -``` - -## Key Features - -### 1. PowerShell Helper Integration - -```xml - - -``` - -The command passes: -- Test assembly output (--list-tests results) -- Test class prefix for filtering -- Collections to skip (optional) -- Output file path - -### 2. Automatic Mode Detection - -```xml - - - <_DetectedMode Condition="$(_FirstLine.StartsWith('collection:'))">collection - <_DetectedMode Condition="$(_FirstLine.StartsWith('class:'))">class - -``` - -### 3. Metadata Generation - -The metadata file includes the detected mode: - -```json -{ - "mode": "collection", // or "class" - "collections": "DatabaseTests;ContainerTests", - ... -} -``` - -## Testing the MSBuild Target - -### Test 1: Project with Collections - -Create a test project with collections: - -```csharp -// Aspire.Hosting.Tests/DatabaseTests.cs -[Collection("DatabaseTests")] -public class PostgresTests { } - -[Collection("DatabaseTests")] -public class MySqlTests { } - -// Aspire.Hosting.Tests/QuickTests.cs -public class QuickTests { } // No collection -``` - -Build: - -```bash -dotnet build tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj \ - /t:Build;ExtractTestClassNames \ - -p:PrepareForHelix=true \ - -p:SplitTestsOnCI=true \ - -p:TestClassNamesPrefix=Aspire.Hosting.Tests \ - /bl:test.binlog -``` - -**Expected Console Output**: -``` -[Aspire.Hosting.Tests] Starting test metadata extraction... -[Aspire.Hosting.Tests] Running discovery helper... -ℹ️ Parsing test assembly output... -🔍 Found collection: DatabaseTests -✅ Detection Results: -ℹ️ Mode: collection -ℹ️ Collections found: 1 -... -[Aspire.Hosting.Tests] Detected mode: collection -[Aspire.Hosting.Tests] Generated entries: 2 -[Aspire.Hosting.Tests] ✅ Test metadata extraction complete! -``` - -**Check Output Files**: - -```bash -# List file -cat artifacts/helix/Aspire.Hosting.Tests.tests.list -# collection:DatabaseTests -# uncollected:* - -# Metadata file -cat artifacts/helix/Aspire.Hosting.Tests.tests.metadata.json | jq .mode -# "collection" -``` - -### Test 2: Project without Collections - -```csharp -// Aspire.Templates.Tests/Test1.cs -public class Test1 { } - -// Aspire.Templates.Tests/Test2.cs -public class Test2 { } -``` - -Build: - -```bash -dotnet build tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj \ - /t:Build;ExtractTestClassNames \ - -p:PrepareForHelix=true \ - -p:SplitTestsOnCI=true \ - -p:TestClassNamesPrefix=Aspire.Templates.Tests -``` - -**Expected Console Output**: -``` -[Aspire.Templates.Tests] Starting test metadata extraction... -[Aspire.Templates.Tests] Running discovery helper... -ℹ️ Parsing test assembly output... -✅ Detection Results: -ℹ️ Mode: class -ℹ️ Test classes found: 12 -... -[Aspire.Templates.Tests] Detected mode: class -[Aspire.Templates.Tests] Generated entries: 12 -[Aspire.Templates.Tests] ✅ Test metadata extraction complete! -``` - -**Check Output Files**: - -```bash -# List file -cat artifacts/helix/Aspire.Templates.Tests.tests.list -# class:Aspire.Templates.Tests.Test1 -# class:Aspire.Templates.Tests.Test2 -# ... - -# Metadata file -cat artifacts/helix/Aspire.Templates.Tests.tests.metadata.json | jq .mode -# "class" -``` - -### Test 3: With Skipped Collections - -```bash -dotnet build tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj \ - /t:Build;ExtractTestClassNames \ - -p:PrepareForHelix=true \ - -p:SplitTestsOnCI=true \ - -p:TestClassNamesPrefix=Aspire.Hosting.Tests \ - -p:TestCollectionsToSkipSplitting="QuickTests;FastTests" -``` - -**Result**: QuickTests and FastTests won't appear in collection list; they'll run in uncollected job. - -## Debugging - -### View Binlog - -```bash -# Install dotnet-binlog if not already installed -dotnet tool install -g dotnet-binlog - -# View the binlog -dotnet-binlog test.binlog -``` - -Look for: -- ExtractTestClassNames target execution -- Console output from test assembly -- PowerShell script execution -- Generated file contents - -### Common Issues - -#### Issue 1: "Discovery helper failed" - -**Symptom**: Target fails with error about missing output file -**Cause**: PowerShell script errored -**Fix**: Check script output in binlog; may need to update regex patterns - -#### Issue 2: "No tests found" - -**Symptom**: Empty .tests.list file -**Cause**: TestClassNamesPrefix doesn't match test namespace -**Fix**: Verify prefix matches actual test namespace - -#### Issue 3: "Mode is empty" - -**Symptom**: `$(_DetectedMode)` is blank -**Cause**: Generated file has unexpected format -**Fix**: Check .tests.list file content manually - -### Manual Verification - -```bash -# Check generated files -ls -la artifacts/helix/*.tests.list -ls -la artifacts/helix/*.tests.metadata.json - -# View contents -cat artifacts/helix/YourProject.Tests.tests.list -cat artifacts/helix/YourProject.Tests.tests.metadata.json | jq . - -# Verify mode detection -cat artifacts/helix/YourProject.Tests.tests.metadata.json | jq -r .mode -# Should output: "collection" or "class" -``` - -## File: `tests/Shared/GetTestProjects.proj` - -### No Changes Needed - -The existing v1 implementation works fine - it just calls MSBuild targets and then the PowerShell matrix generator. - -```xml - - - $(MSBuildThisFileDirectory)..\..\ - $(ArtifactsDir)test-matrices\ - - - - - - - - - - - - - - - -``` - -## Next Steps - -Proceed to [Step 3: Matrix Generator (v3)](./STEP_03_MATRIX_GENERATOR_V3.md) \ No newline at end of file diff --git a/docs/test-splitting/STEP_02_POWERSHELL_SCRIPT.md b/docs/test-splitting/STEP_02_POWERSHELL_SCRIPT.md deleted file mode 100644 index c817c46f40d..00000000000 --- a/docs/test-splitting/STEP_02_POWERSHELL_SCRIPT.md +++ /dev/null @@ -1,374 +0,0 @@ -# Step 2: PowerShell Matrix Generation Script - -## Overview - -Create a cross-platform PowerShell script that reads test class lists and generates JSON matrices for CI consumption. - -## File: `eng/scripts/generate-test-matrix.ps1` - -### Complete Implementation - -```powershell -<# -.SYNOPSIS - Generates CI test matrices from test class enumeration files. - -.DESCRIPTION - This script reads .tests.list and .tests.metadata.json files produced by the - ExtractTestClassNames MSBuild target and generates a JSON matrix file for - consumption by GitHub Actions or Azure DevOps. - - The script is cross-platform and runs on Windows, Linux, and macOS. - -.PARAMETER TestListsDirectory - Directory containing .tests.list and .tests.metadata.json files. - Typically: artifacts/helix/ - -.PARAMETER OutputDirectory - Directory where the JSON matrix file will be written. - Typically: artifacts/test-matrices/ - -.PARAMETER BuildOs - Current operating system being built for (windows, linux, darwin). - Used for logging and debugging. - -.EXAMPLE - pwsh generate-test-matrix.ps1 -TestListsDirectory ./artifacts/helix -OutputDirectory ./artifacts/matrices -BuildOs linux - -.NOTES - Author: Aspire Team - Date: 2025-10-16 - Requires: PowerShell 7.0+ (cross-platform) -#> - -[CmdletBinding()] -param( - [Parameter(Mandatory=$true, HelpMessage="Directory containing test list files")] - [ValidateScript({Test-Path $_ -PathType Container})] - [string]$TestListsDirectory, - - [Parameter(Mandatory=$true, HelpMessage="Output directory for matrix JSON")] - [string]$OutputDirectory, - - [Parameter(Mandatory=$false, HelpMessage="Current OS: windows, linux, or darwin")] - [ValidateSet('windows', 'linux', 'darwin', '')] - [string]$BuildOs = '' -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version Latest - -#region Helper Functions - -function Write-Message { - <# - .SYNOPSIS - Writes a formatted message to the console. - #> - param( - [Parameter(Mandatory=$true)] - [AllowEmptyString()] - [string]$Message, - - [Parameter(Mandatory=$false)] - [ValidateSet('Info', 'Success', 'Warning', 'Error')] - [string]$Level = 'Info' - ) - - $prefix = switch ($Level) { - 'Success' { '✅' } - 'Warning' { '⚠️' } - 'Error' { '❌' } - default { 'ℹ️' } - } - - $color = switch ($Level) { - 'Success' { 'Green' } - 'Warning' { 'Yellow' } - 'Error' { 'Red' } - default { 'Cyan' } - } - - Write-Host "$prefix $Message" -ForegroundColor $color -} - -function Get-TestListFiles { - <# - .SYNOPSIS - Finds all .tests.list files in the specified directory. - #> - param([string]$Directory) - - Get-ChildItem -Path $Directory -Filter "*.tests.list" -Recurse -ErrorAction SilentlyContinue -} - -function Read-TestMetadata { - <# - .SYNOPSIS - Reads and parses test metadata JSON file. - #> - param( - [string]$MetadataFile, - [string]$ProjectName - ) - - # Default metadata values - $defaults = @{ - testClassNamesPrefix = $ProjectName - testProjectPath = "tests/$ProjectName/$ProjectName.csproj" - requiresNugets = 'false' - requiresTestSdk = 'false' - testSessionTimeout = '20m' - testHangTimeout = '10m' - enablePlaywrightInstall = 'false' - } - - if (-not (Test-Path $MetadataFile)) { - Write-Message "No metadata file found for $ProjectName, using defaults" -Level Warning - return $defaults - } - - try { - $content = Get-Content $MetadataFile -Raw | ConvertFrom-Json - - # Merge with defaults (content overrides defaults) - foreach ($key in $content.PSObject.Properties.Name) { - $defaults[$key] = $content.$key - } - - return $defaults - } - catch { - Write-Message "Failed to parse metadata for ${ProjectName}: $_" -Level Warning - return $defaults - } -} - -function New-MatrixEntry { - <# - .SYNOPSIS - Creates a matrix entry object for a test class. - #> - param( - [string]$FullClassName, - [string]$ProjectName, - [hashtable]$Metadata - ) - - $prefix = $Metadata.testClassNamesPrefix - $shortname = $FullClassName - - # Strip prefix if present (e.g., "Aspire.Templates.Tests.MyClass" → "MyClass") - if ($prefix -and $FullClassName.StartsWith("$prefix.")) { - $shortname = $FullClassName.Substring($prefix.Length + 1) - } - - # Return ordered hashtable for consistent JSON output - [ordered]@{ - shortname = $shortname - projectName = $ProjectName - fullClassName = $FullClassName - testProjectPath = $Metadata.testProjectPath - requiresNugets = ($Metadata.requiresNugets -eq 'true') - requiresTestSdk = ($Metadata.requiresTestSdk -eq 'true') - testSessionTimeout = $Metadata.testSessionTimeout - testHangTimeout = $Metadata.testHangTimeout - enablePlaywrightInstall = ($Metadata.enablePlaywrightInstall -eq 'true') - } -} - -#endregion - -#region Main Script - -Write-Message "Starting matrix generation for BuildOs=$BuildOs" -Write-Message "Test lists directory: $TestListsDirectory" -Write-Message "Output directory: $OutputDirectory" - -# Find all test list files -$listFiles = Get-TestListFiles -Directory $TestListsDirectory - -if ($listFiles.Count -eq 0) { - Write-Message "No test list files found in $TestListsDirectory" -Level Warning - Write-Message "Creating empty matrix file..." - - # Create empty matrix - $emptyMatrix = @{ include = @() } - $outputFile = Join-Path $OutputDirectory "split-tests-matrix.json" - - # Ensure output directory exists - if (-not (Test-Path $OutputDirectory)) { - New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null - } - - $emptyMatrix | ConvertTo-Json -Depth 10 -Compress | Set-Content -Path $outputFile -Encoding UTF8 - Write-Message "Created empty matrix: $outputFile" -Level Success - exit 0 -} - -Write-Message "Found $($listFiles.Count) test list file(s)" -Level Success - -# Process each test list file -$allEntries = [System.Collections.ArrayList]::new() -$stats = @{} - -foreach ($listFile in $listFiles) { - # Extract project name (e.g., "Aspire.Templates.Tests.tests.list" → "Aspire.Templates.Tests") - $projectName = [System.IO.Path]::GetFileNameWithoutExtension($listFile.Name -replace '\.tests$', '') - - Write-Message "Processing $projectName..." - - # Read test class names - $classes = Get-Content $listFile.FullName | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } - - if ($classes.Count -eq 0) { - Write-Message " No test classes found, skipping" -Level Warning - continue - } - - # Read metadata - $metadataFile = $listFile.FullName -replace '\.tests\.list$', '.tests.metadata.json' - $metadata = Read-TestMetadata -MetadataFile $metadataFile -ProjectName $projectName - - # Generate matrix entry for each test class - $projectEntryCount = 0 - foreach ($class in $classes) { - $entry = New-MatrixEntry -FullClassName $class -ProjectName $projectName -Metadata $metadata - [void]$allEntries.Add($entry) - $projectEntryCount++ - } - - $stats[$projectName] = $projectEntryCount - Write-Message " Added $projectEntryCount test class(es)" -Level Success -} - -# Generate final matrix -$matrix = @{ - include = $allEntries.ToArray() -} - -# Write JSON file -$outputFile = Join-Path $OutputDirectory "split-tests-matrix.json" - -# Ensure output directory exists -if (-not (Test-Path $OutputDirectory)) { - New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null -} - -$jsonOutput = $matrix | ConvertTo-Json -Depth 10 -Compress -$jsonOutput | Set-Content -Path $outputFile -Encoding UTF8 -NoNewline - -Write-Message "" -Write-Message "Generated matrix with $($allEntries.Count) total test(s)" -Level Success -Write-Message "Output file: $outputFile" -Level Success -Write-Message "" -Write-Message "Matrix breakdown by project:" -Level Info - -foreach ($proj in $stats.Keys | Sort-Object) { - Write-Message " $proj`: $($stats[$proj]) class(es)" -Level Info -} - -Write-Message "" -Write-Message "Matrix generation complete! ✨" -Level Success - -#endregion -``` - -## Script Features - -### Cross-Platform Compatibility - -- ✅ Uses `System.IO.Path` for path operations -- ✅ No OS-specific cmdlets -- ✅ Tested on Windows, Linux, macOS -- ✅ UTF-8 encoding for JSON output - -### Error Handling - -- Validates input directory exists -- Handles missing metadata gracefully (uses defaults) -- Creates empty matrix if no tests found (CI won't fail) -- Detailed error messages - -### Logging - -- Color-coded output (Info, Success, Warning, Error) -- Shows progress per project -- Summary statistics at end -- Helpful for debugging CI issues - -## Testing the Script - -### Test 1: Empty Directory - -```powershell -# Should create empty matrix without errors -mkdir test-empty -pwsh eng/scripts/generate-test-matrix.ps1 ` - -TestListsDirectory ./test-empty ` - -OutputDirectory ./test-output ` - -BuildOs linux -``` - -**Expected**: Creates `split-tests-matrix.json` with `{"include":[]}` - -### Test 2: With Test Lists - -```powershell -# First, build a split test project to generate .tests.list files -dotnet build tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj ` - /t:Build;ExtractTestClassNames ` - -p:PrepareForHelix=true ` - -p:SplitTestsForCI=true ` - -p:TestClassNamesPrefix=Aspire.Templates.Tests - -# Then run the script -pwsh eng/scripts/generate-test-matrix.ps1 ` - -TestListsDirectory ./artifacts/helix ` - -OutputDirectory ./artifacts/test-matrices ` - -BuildOs linux -``` - -**Expected**: -- Creates matrix with ~10-15 entries -- Each entry has all required fields -- Valid JSON - -### Test 3: Verify JSON Structure - -```powershell -# Load and inspect the generated matrix -$matrix = Get-Content ./artifacts/test-matrices/split-tests-matrix.json | ConvertFrom-Json - -# Check structure -$matrix.include.Count # Should be > 0 -$matrix.include[0].PSObject.Properties.Name # Should show all fields - -# Verify required fields -$matrix.include | ForEach-Object { - if (-not $_.shortname) { Write-Error "Missing shortname" } - if (-not $_.fullClassName) { Write-Error "Missing fullClassName" } - if (-not $_.projectName) { Write-Error "Missing projectName" } -} -``` - -## Common Issues - -### Issue 1: "Cannot find path" - -**Cause**: TestListsDirectory doesn't exist -**Fix**: Ensure the directory is created before running script - -### Issue 2: Invalid JSON - -**Cause**: Special characters in class names -**Fix**: PowerShell's `ConvertTo-Json` handles this automatically - -### Issue 3: Empty matrix but tests exist - -**Cause**: `.tests.list` files not in expected location -**Fix**: Check `artifacts/helix/` directory structure - -## Next Steps - -Proceed to [Step 3: GitHub Actions Integration](./STEP_03_GITHUB_ACTIONS.md) \ No newline at end of file diff --git a/docs/test-splitting/STEP_02_POWERSHELL_SCRIPT_V2.md b/docs/test-splitting/STEP_02_POWERSHELL_SCRIPT_V2.md deleted file mode 100644 index 5d92131f6fb..00000000000 --- a/docs/test-splitting/STEP_02_POWERSHELL_SCRIPT_V2.md +++ /dev/null @@ -1,708 +0,0 @@ -# Step 2: PowerShell Script Implementation (v2 - Collection Support) - -## Overview - -Enhanced PowerShell script that reads collection-based test lists and generates a matrix with: -- One entry per collection -- One entry for all uncollected tests - -## File: `eng/scripts/generate-test-matrix.ps1` - -### Complete Implementation - -```powershell -<# -.SYNOPSIS - Generates CI test matrices from collection-based test enumeration files. - -.DESCRIPTION - This script reads .tests.list and .tests.metadata.json files produced by the - ExtractTestClassNames MSBuild target and generates a JSON matrix file for - consumption by GitHub Actions or Azure DevOps. - - Supports both xUnit collections (grouped tests) and uncollected tests (catch-all). - - The script is cross-platform and runs on Windows, Linux, and macOS. - -.PARAMETER TestListsDirectory - Directory containing .tests.list and .tests.metadata.json files. - Typically: artifacts/helix/ - -.PARAMETER OutputDirectory - Directory where the JSON matrix file will be written. - Typically: artifacts/test-matrices/ - -.PARAMETER BuildOs - Current operating system being built for (windows, linux, darwin). - Used for logging and debugging. - -.EXAMPLE - pwsh generate-test-matrix.ps1 -TestListsDirectory ./artifacts/helix -OutputDirectory ./artifacts/matrices -BuildOs linux - -.NOTES - Author: Aspire Team - Date: 2025-10-16 - Version: 2.0 (Collection-based splitting) - Requires: PowerShell 7.0+ (cross-platform) -#> - -[CmdletBinding()] -param( - [Parameter(Mandatory=$true, HelpMessage="Directory containing test list files")] - [ValidateScript({Test-Path $_ -PathType Container})] - [string]$TestListsDirectory, - - [Parameter(Mandatory=$true, HelpMessage="Output directory for matrix JSON")] - [string]$OutputDirectory, - - [Parameter(Mandatory=$false, HelpMessage="Current OS: windows, linux, or darwin")] - [ValidateSet('windows', 'linux', 'darwin', '')] - [string]$BuildOs = '' -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version Latest - -#region Helper Functions - -function Write-Message { - <# - .SYNOPSIS - Writes a formatted message to the console. - #> - param( - [Parameter(Mandatory=$true)] - [AllowEmptyString()] - [string]$Message, - - [Parameter(Mandatory=$false)] - [ValidateSet('Info', 'Success', 'Warning', 'Error', 'Debug')] - [string]$Level = 'Info' - ) - - $prefix = switch ($Level) { - 'Success' { '✅' } - 'Warning' { '⚠️' } - 'Error' { '❌' } - 'Debug' { '🔍' } - default { 'ℹ️' } - } - - $color = switch ($Level) { - 'Success' { 'Green' } - 'Warning' { 'Yellow' } - 'Error' { 'Red' } - 'Debug' { 'Gray' } - default { 'Cyan' } - } - - Write-Host "$prefix $Message" -ForegroundColor $color -} - -function Get-TestListFiles { - <# - .SYNOPSIS - Finds all .tests.list files in the specified directory. - #> - param([string]$Directory) - - Get-ChildItem -Path $Directory -Filter "*.tests.list" -Recurse -ErrorAction SilentlyContinue -} - -function Read-TestMetadata { - <# - .SYNOPSIS - Reads and parses test metadata JSON file. - #> - param( - [string]$MetadataFile, - [string]$ProjectName - ) - - # Default metadata values - $defaults = @{ - testClassNamesPrefix = $ProjectName - testProjectPath = "tests/$ProjectName/$ProjectName.csproj" - collections = '' - requiresNugets = 'false' - requiresTestSdk = 'false' - testSessionTimeout = '20m' - testHangTimeout = '10m' - uncollectedTestsSessionTimeout = '15m' - uncollectedTestsHangTimeout = '8m' - enablePlaywrightInstall = 'false' - } - - if (-not (Test-Path $MetadataFile)) { - Write-Message "No metadata file found for $ProjectName, using defaults" -Level Warning - return $defaults - } - - try { - $content = Get-Content $MetadataFile -Raw | ConvertFrom-Json - - # Merge with defaults (content overrides defaults) - foreach ($key in $content.PSObject.Properties.Name) { - $defaults[$key] = $content.$key - } - - return $defaults - } - catch { - Write-Message "Failed to parse metadata for ${ProjectName}: $_" -Level Warning - return $defaults - } -} - -function Get-CollectionFilterArg { - <# - .SYNOPSIS - Generates xUnit filter argument for a specific collection. - #> - param([string]$CollectionName) - - return "--filter-collection `"$CollectionName`"" -} - -function Get-UncollectedFilterArg { - <# - .SYNOPSIS - Generates xUnit filter argument to exclude all collections. - #> - param([string[]]$Collections) - - if ($Collections.Count -eq 0) { - # No collections to exclude - run all tests - return "" - } - - # Build filter to exclude all collections - $filters = $Collections | ForEach-Object { - "--filter-not-collection `"$_`"" - } - - return $filters -join ' ' -} - -function New-CollectionMatrixEntry { - <# - .SYNOPSIS - Creates a matrix entry for a collection. - #> - param( - [string]$CollectionName, - [string]$ProjectName, - [hashtable]$Metadata - ) - - $filterArg = Get-CollectionFilterArg -CollectionName $CollectionName - - # Check for per-collection timeout overrides - $collectionTimeoutKey = "TestCollection_${CollectionName}_SessionTimeout" - $collectionHangTimeoutKey = "TestCollection_${CollectionName}_HangTimeout" - - $sessionTimeout = $Metadata.testSessionTimeout - $hangTimeout = $Metadata.testHangTimeout - - # Per-collection timeouts would come from metadata if specified - # For now, use project defaults - - [ordered]@{ - type = "collection" - name = $CollectionName - shortname = "Collection_$CollectionName" - projectName = $ProjectName - testProjectPath = $Metadata.testProjectPath - filterArg = $filterArg - requiresNugets = ($Metadata.requiresNugets -eq 'true') - requiresTestSdk = ($Metadata.requiresTestSdk -eq 'true') - testSessionTimeout = $sessionTimeout - testHangTimeout = $hangTimeout - enablePlaywrightInstall = ($Metadata.enablePlaywrightInstall -eq 'true') - } -} - -function New-UncollectedMatrixEntry { - <# - .SYNOPSIS - Creates a matrix entry for uncollected tests. - #> - param( - [string[]]$Collections, - [string]$ProjectName, - [hashtable]$Metadata - ) - - $filterArg = Get-UncollectedFilterArg -Collections $Collections - - # Use specific timeouts for uncollected tests (usually faster) - $sessionTimeout = if ($Metadata.uncollectedTestsSessionTimeout) { - $Metadata.uncollectedTestsSessionTimeout - } else { - $Metadata.testSessionTimeout - } - - $hangTimeout = if ($Metadata.uncollectedTestsHangTimeout) { - $Metadata.uncollectedTestsHangTimeout - } else { - $Metadata.testHangTimeout - } - - [ordered]@{ - type = "uncollected" - name = "UncollectedTests" - shortname = "Uncollected" - projectName = $ProjectName - testProjectPath = $Metadata.testProjectPath - filterArg = $filterArg - requiresNugets = ($Metadata.requiresNugets -eq 'true') - requiresTestSdk = ($Metadata.requiresTestSdk -eq 'true') - testSessionTimeout = $sessionTimeout - testHangTimeout = $hangTimeout - enablePlaywrightInstall = ($Metadata.enablePlaywrightInstall -eq 'true') - } -} - -function Parse-TestListFile { - <# - .SYNOPSIS - Parses a .tests.list file and returns collections and flags. - #> - param([string]$FilePath) - - $lines = Get-Content $FilePath | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } - - $result = @{ - Collections = [System.Collections.ArrayList]::new() - HasUncollected = $false - } - - foreach ($line in $lines) { - if ($line -match '^collection:(.+)$') { - [void]$result.Collections.Add($Matches[1].Trim()) - } - elseif ($line -match '^uncollected:') { - $result.HasUncollected = $true - } - } - - return $result -} - -#endregion - -#region Main Script - -Write-Message "Starting collection-based matrix generation for BuildOs=$BuildOs" -Write-Message "Test lists directory: $TestListsDirectory" -Write-Message "Output directory: $OutputDirectory" - -# Find all test list files -$listFiles = Get-TestListFiles -Directory $TestListsDirectory - -if ($listFiles.Count -eq 0) { - Write-Message "No test list files found in $TestListsDirectory" -Level Warning - Write-Message "Creating empty matrix file..." - - # Create empty matrix - $emptyMatrix = @{ include = @() } - $outputFile = Join-Path $OutputDirectory "split-tests-matrix.json" - - # Ensure output directory exists - if (-not (Test-Path $OutputDirectory)) { - New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null - } - - $emptyMatrix | ConvertTo-Json -Depth 10 -Compress | Set-Content -Path $outputFile -Encoding UTF8 - Write-Message "Created empty matrix: $outputFile" -Level Success - exit 0 -} - -Write-Message "Found $($listFiles.Count) test list file(s)" -Level Success - -# Process each test list file -$allEntries = [System.Collections.ArrayList]::new() -$stats = @{} - -foreach ($listFile in $listFiles) { - # Extract project name - $projectName = [System.IO.Path]::GetFileNameWithoutExtension($listFile.Name -replace '\.tests$', '') - - Write-Message "" - Write-Message "Processing $projectName..." -Level Info - - # Parse test list file - $parsed = Parse-TestListFile -FilePath $listFile.FullName - - if ($parsed.Collections.Count -eq 0 -and -not $parsed.HasUncollected) { - Write-Message " No collections or uncollected tests found, skipping" -Level Warning - continue - } - - # Read metadata - $metadataFile = $listFile.FullName -replace '\.tests\.list$', '.tests.metadata.json' - $metadata = Read-TestMetadata -MetadataFile $metadataFile -ProjectName $projectName - - $projectStats = @{ - Collections = 0 - Uncollected = 0 - } - - # Generate matrix entries for each collection - foreach ($collectionName in $parsed.Collections) { - Write-Message " Found collection: $collectionName" -Level Debug - - $entry = New-CollectionMatrixEntry ` - -CollectionName $collectionName ` - -ProjectName $projectName ` - -Metadata $metadata - - [void]$allEntries.Add($entry) - $projectStats.Collections++ - } - - # Generate matrix entry for uncollected tests - if ($parsed.HasUncollected) { - Write-Message " Adding uncollected tests job" -Level Debug - - $entry = New-UncollectedMatrixEntry ` - -Collections $parsed.Collections.ToArray() ` - -ProjectName $projectName ` - -Metadata $metadata - - [void]$allEntries.Add($entry) - $projectStats.Uncollected = 1 - } - - $stats[$projectName] = $projectStats - - $totalJobs = $projectStats.Collections + $projectStats.Uncollected - Write-Message " ✅ Generated $totalJobs job(s): $($projectStats.Collections) collection(s) + $($projectStats.Uncollected) uncollected" -Level Success -} - -# Generate final matrix -$matrix = @{ - include = $allEntries.ToArray() -} - -# Write JSON file -$outputFile = Join-Path $OutputDirectory "split-tests-matrix.json" - -# Ensure output directory exists -if (-not (Test-Path $OutputDirectory)) { - New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null -} - -$jsonOutput = $matrix | ConvertTo-Json -Depth 10 -Compress -$jsonOutput | Set-Content -Path $outputFile -Encoding UTF8 -NoNewline - -# Summary -Write-Message "" -Write-Message ("=" * 60) -Level Info -Write-Message "Matrix Generation Summary" -Level Success -Write-Message ("=" * 60) -Level Info -Write-Message "" -Write-Message "Total Jobs: $($allEntries.Count)" -Level Success -Write-Message "Output File: $outputFile" -Level Success -Write-Message "" -Write-Message "Breakdown by Project:" -Level Info - -foreach ($proj in $stats.Keys | Sort-Object) { - $s = $stats[$proj] - $collText = if ($s.Collections -eq 1) { "collection" } else { "collections" } - $uncText = if ($s.Uncollected -eq 1) { "uncollected job" } else { "uncollected jobs" } - - Write-Message " $proj`: $($s.Collections) $collText + $($s.Uncollected) $uncText" -Level Info -} - -Write-Message "" -Write-Message "Matrix generation complete! ✨" -Level Success - -#endregion -``` - -## Key Features of v2 Script - -### 1. Collection Parsing - -```powershell -function Parse-TestListFile { - # Parses format: - # collection:DatabaseTests - # collection:ContainerTests - # uncollected:* - - foreach ($line in $lines) { - if ($line -match '^collection:(.+)$') { - # Extract collection name - } - elseif ($line -match '^uncollected:') { - # Flag that uncollected tests exist - } - } -} -``` - -### 2. Filter Generation - -```powershell -# For a collection -"--filter-collection `"DatabaseTests`"" - -# For uncollected (exclude all collections) -"--filter-not-collection `"DatabaseTests`" --filter-not-collection `"ContainerTests`"" -``` - -### 3. Smart Timeouts - -```powershell -# Collections use project-level timeouts (usually longer) -$sessionTimeout = $Metadata.testSessionTimeout # e.g., 25m - -# Uncollected uses shorter timeouts (fast tests) -$sessionTimeout = $Metadata.uncollectedTestsSessionTimeout # e.g., 15m -``` - -### 4. Matrix Entry Types - -```powershell -# Collection entry -@{ - type = "collection" - name = "DatabaseTests" - filterArg = "--filter-collection `"DatabaseTests`"" - # ... -} - -# Uncollected entry -@{ - type = "uncollected" - name = "UncollectedTests" - filterArg = "--filter-not-collection `"DatabaseTests`" ..." - # ... -} -``` - -## Testing the Script - -### Test 1: Project with No Collections - -Create a test list file: - -```bash -# artifacts/helix/SomeProject.Tests.tests.list -uncollected:* -``` - -Create metadata: - -```json -{ - "projectName": "SomeProject.Tests", - "testProjectPath": "tests/SomeProject.Tests/SomeProject.Tests.csproj", - "collections": "", - "testSessionTimeout": "20m", - "testHangTimeout": "10m", - "uncollectedTestsSessionTimeout": "15m", - "uncollectedTestsHangTimeout": "8m" -} -``` - -Run script: - -```powershell -pwsh eng/scripts/generate-test-matrix.ps1 ` - -TestListsDirectory ./artifacts/helix ` - -OutputDirectory ./artifacts/test-matrices ` - -BuildOs linux -``` - -**Expected Output**: -```json -{ - "include": [ - { - "type": "uncollected", - "name": "UncollectedTests", - "shortname": "Uncollected", - "projectName": "SomeProject.Tests", - "testProjectPath": "tests/SomeProject.Tests/SomeProject.Tests.csproj", - "filterArg": "", - "requiresNugets": false, - "requiresTestSdk": false, - "testSessionTimeout": "15m", - "testHangTimeout": "8m", - "enablePlaywrightInstall": false - } - ] -} -``` - -**Result**: 1 job - -### Test 2: Project with Collections - -Create test list: - -```bash -# artifacts/helix/Aspire.Hosting.Tests.tests.list -collection:DatabaseTests -collection:ContainerTests -uncollected:* -``` - -Create metadata: - -```json -{ - "projectName": "Aspire.Hosting.Tests", - "testProjectPath": "tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj", - "collections": "DatabaseTests;ContainerTests", - "testSessionTimeout": "25m", - "testHangTimeout": "12m", - "uncollectedTestsSessionTimeout": "15m", - "uncollectedTestsHangTimeout": "8m", - "requiresNugets": "false", - "requiresTestSdk": "false", - "enablePlaywrightInstall": "false" -} -``` - -Run script: - -```powershell -pwsh eng/scripts/generate-test-matrix.ps1 ` - -TestListsDirectory ./artifacts/helix ` - -OutputDirectory ./artifacts/test-matrices ` - -BuildOs linux -``` - -**Expected Output**: - -```json -{ - "include": [ - { - "type": "collection", - "name": "DatabaseTests", - "shortname": "Collection_DatabaseTests", - "projectName": "Aspire.Hosting.Tests", - "filterArg": "--filter-collection \"DatabaseTests\"", - "testSessionTimeout": "25m", - "testHangTimeout": "12m", - ... - }, - { - "type": "collection", - "name": "ContainerTests", - "shortname": "Collection_ContainerTests", - "projectName": "Aspire.Hosting.Tests", - "filterArg": "--filter-collection \"ContainerTests\"", - "testSessionTimeout": "25m", - "testHangTimeout": "12m", - ... - }, - { - "type": "uncollected", - "name": "UncollectedTests", - "shortname": "Uncollected", - "projectName": "Aspire.Hosting.Tests", - "filterArg": "--filter-not-collection \"DatabaseTests\" --filter-not-collection \"ContainerTests\"", - "testSessionTimeout": "15m", - "testHangTimeout": "8m", - ... - } - ] -} -``` - -**Result**: 3 jobs - -### Test 3: Verify Filter Arguments - -Load and inspect the matrix: - -```powershell -$matrix = Get-Content ./artifacts/test-matrices/split-tests-matrix.json | ConvertFrom-Json - -# Check collection filters -$matrix.include | Where-Object { $_.type -eq 'collection' } | ForEach-Object { - Write-Host "$($_.name): $($_.filterArg)" -} - -# Check uncollected filter -$uncollected = $matrix.include | Where-Object { $_.type -eq 'uncollected' } -Write-Host "Uncollected: $($uncollected.filterArg)" -``` - -**Expected Console Output**: -``` -DatabaseTests: --filter-collection "DatabaseTests" -ContainerTests: --filter-collection "ContainerTests" -Uncollected: --filter-not-collection "DatabaseTests" --filter-not-collection "ContainerTests" -``` - -### Test 4: Multiple Projects - -Create test lists for multiple projects: - -```bash -# artifacts/helix/Aspire.Hosting.Tests.tests.list -collection:DatabaseTests -uncollected:* - -# artifacts/helix/Aspire.Templates.Tests.tests.list -collection:StarterTemplate -collection:BasicTemplate -uncollected:* -``` - -Run script: - -```powershell -pwsh eng/scripts/generate-test-matrix.ps1 ` - -TestListsDirectory ./artifacts/helix ` - -OutputDirectory ./artifacts/test-matrices ` - -BuildOs linux -``` - -**Expected Result**: 6 jobs total -- 2 from Aspire.Hosting.Tests (1 collection + 1 uncollected) -- 4 from Aspire.Templates.Tests (2 collections + 1 uncollected) - -## Validation Checklist - -- [ ] Script runs without errors on all 3 OSes -- [ ] Empty directory creates empty matrix -- [ ] Single uncollected entry creates 1 job -- [ ] Collections create separate jobs -- [ ] Uncollected filter excludes all collections -- [ ] Metadata defaults work when file missing -- [ ] JSON output is valid and parseable -- [ ] Filter arguments have correct syntax -- [ ] Timeouts are applied correctly -- [ ] Summary statistics are accurate - -## Common Issues & Solutions - -### Issue 1: "Collection not found" in test output - -**Symptom**: xunit can't find collection name -**Cause**: Collection name has special characters or spaces -**Fix**: Escape collection names in filter arguments (already handled with quotes) - -### Issue 2: Uncollected filter too long - -**Symptom**: Command line too long with many collections -**Cause**: Too many `--filter-not-collection` arguments -**Fix**: Consider regrouping collections or using different approach - -### Issue 3: Empty uncollected job - -**Symptom**: Uncollected job runs but no tests execute -**Cause**: All tests are in collections -**Fix**: This is OK - job will exit with code 8 (zero tests), which we ignore - -## Next Steps - -Proceed to [Step 4: Project Configuration (v2)](./STEP_04_PROJECT_CONFIG_V2.md) - GitHub Actions doesn't need changes since it just consumes the matrix! \ No newline at end of file diff --git a/docs/test-splitting/STEP_03_GITHUB_ACTIONS.md b/docs/test-splitting/STEP_03_GITHUB_ACTIONS.md deleted file mode 100644 index 78d70269c81..00000000000 --- a/docs/test-splitting/STEP_03_GITHUB_ACTIONS.md +++ /dev/null @@ -1,414 +0,0 @@ -# Step 3: GitHub Actions Integration - -## Overview - -Update GitHub Actions workflows to use the new MSBuild-based matrix generation while maintaining full support for all 3 OSes. - -## Critical Requirement: Per-OS Matrix Generation - -**Each OS MUST generate its own matrix** because: -1. Projects can opt-in/out per OS (`RunOnGithubActionsWindows`, etc.) -2. Some tests only run on specific OSes (e.g., Docker tests on Linux) -3. File path differences between OSes -4. Test discovery may differ per platform - -## File: `.github/actions/enumerate-tests/action.yml` - -### Complete Replacement - -```yaml -name: 'Enumerate test projects' -description: 'Enumerate test projects and generate test matrices for the current OS' -inputs: - includeIntegrations: - description: 'Include integration tests in enumeration' - required: false - type: boolean - default: false - includeSplitTests: - description: 'Include and generate split test matrices' - required: false - type: boolean - default: false - -outputs: - integrations_tests_matrix: - description: 'JSON matrix of integration test projects' - value: ${{ steps.load_integrations_matrix.outputs.matrix }} - split_tests_matrix: - description: 'JSON matrix of split test classes' - value: ${{ steps.load_split_matrix.outputs.matrix }} - -runs: - using: "composite" - steps: - - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Set up .NET Core - uses: actions/setup-dotnet@3951f0dfe7a07e2313ec93c75700083e2005cbab # v4.3.0 - with: - global-json-file: ${{ github.workspace }}/global.json - - - name: Generate test project lists - if: ${{ inputs.includeIntegrations }} - shell: pwsh - run: > - dotnet build ${{ github.workspace }}/tests/Shared/GetTestProjects.proj - /bl:${{ github.workspace }}/artifacts/log/Debug/GetTestProjects.binlog - /p:TestsListOutputPath=${{ github.workspace }}/artifacts/TestsForGithubActions.list - /p:TestMatrixOutputPath=${{ github.workspace }}/artifacts/test-matrices/ - /p:ContinuousIntegrationBuild=true - - - name: Build split test projects - if: ${{ inputs.includeSplitTests }} - shell: pwsh - run: | - $ErrorActionPreference = 'Stop' - - $splitProjectsFile = "${{ github.workspace }}/artifacts/TestsForGithubActions.list.split-projects" - - if (-not (Test-Path $splitProjectsFile)) { - Write-Host "::notice::No split test projects found for ${{ runner.os }}" - exit 0 - } - - $splitProjects = Get-Content $splitProjectsFile | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } - - if ($splitProjects.Count -eq 0) { - Write-Host "::notice::No split test projects to build for ${{ runner.os }}" - exit 0 - } - - Write-Host "::group::Building $($splitProjects.Count) split test project(s) for ${{ runner.os }}" - - foreach ($shortname in $splitProjects) { - Write-Host "Processing $shortname..." - - # Find the project file (try both naming patterns) - $projectPath1 = "${{ github.workspace }}/tests/$shortname.Tests/$shortname.Tests.csproj" - $projectPath2 = "${{ github.workspace }}/tests/Aspire.$shortname.Tests/Aspire.$shortname.Tests.csproj" - - if (Test-Path $projectPath1) { - $projectPath = $projectPath1 - } elseif (Test-Path $projectPath2) { - $projectPath = $projectPath2 - } else { - Write-Error "::error::Could not find project for $shortname" - exit 1 - } - - Write-Host " Building: $projectPath" - - # Build with ExtractTestClassNames target - dotnet build $projectPath ` - /t:Build`;ExtractTestClassNames ` - /bl:${{ github.workspace }}/artifacts/log/Debug/Build_$shortname.binlog ` - -p:PrepareForHelix=true ` - -p:SplitTestsForCI=true ` - -p:InstallBrowsersForPlaywright=false - - if ($LASTEXITCODE -ne 0) { - Write-Error "::error::Build failed for $shortname with exit code $LASTEXITCODE" - exit $LASTEXITCODE - } - - Write-Host " ✅ Successfully built $shortname" - } - - Write-Host "::endgroup::" - Write-Host "::notice::Successfully built all $($splitProjects.Count) split test projects for ${{ runner.os }}" - - - name: Load integrations matrix - id: load_integrations_matrix - if: ${{ inputs.includeIntegrations }} - shell: pwsh - run: | - $filePath = "${{ github.workspace }}/artifacts/TestsForGithubActions.list" - - if (-not (Test-Path $filePath)) { - Write-Error "::error::Test list file not found: $filePath" - exit 1 - } - - $lines = Get-Content $filePath | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } - - $matrix = @{ - shortname = $lines | Sort-Object - } - - $json = $matrix | ConvertTo-Json -Compress - - Write-Host "::notice::Generated integrations matrix for ${{ runner.os }} with $($lines.Count) project(s)" - "matrix=$json" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - - - name: Load split tests matrix - id: load_split_matrix - if: ${{ inputs.includeSplitTests }} - shell: pwsh - run: | - $matrixFile = "${{ github.workspace }}/artifacts/test-matrices/split-tests-matrix.json" - - if (Test-Path $matrixFile) { - $json = Get-Content $matrixFile -Raw - $matrix = $json | ConvertFrom-Json - - $testCount = if ($matrix.include) { $matrix.include.Count } else { 0 } - - Write-Host "::notice::Generated split tests matrix for ${{ runner.os }} with $testCount test(s)" - "matrix=$json" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - } else { - Write-Host "::notice::No split tests matrix found for ${{ runner.os }}, using empty matrix" - $emptyMatrix = @{ include = @() } | ConvertTo-Json -Compress - "matrix=$emptyMatrix" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - } - - - name: Upload artifacts - if: always() - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 - with: - name: logs-enumerate-tests-${{ runner.os }} - path: | - artifacts/log/**/*.binlog - artifacts/**/*.list - artifacts/**/*.metadata.json - artifacts/test-matrices/**/*.json - if-no-files-found: warn -``` - -## File: `.github/workflows/tests.yml` - -### Modified Sections - -#### 1. Update setup jobs (KEEP SEPARATE PER OS) - -```yaml -jobs: - # IMPORTANT: Keep separate setup jobs for each OS - # Each OS generates its own matrix because projects can opt-in/out per OS - - setup_for_tests_lin: - name: Setup for tests (Linux) - runs-on: ubuntu-latest - outputs: - integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }} - split_tests_matrix: ${{ steps.generate_tests_matrix.outputs.split_tests_matrix }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - uses: ./.github/actions/enumerate-tests - id: generate_tests_matrix - with: - includeIntegrations: true - includeSplitTests: true # NEW: Enable split tests - - setup_for_tests_macos: - name: Setup for tests (macOS) - runs-on: macos-latest - outputs: - integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }} - split_tests_matrix: ${{ steps.generate_tests_matrix.outputs.split_tests_matrix }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - uses: ./.github/actions/enumerate-tests - id: generate_tests_matrix - with: - includeIntegrations: true - includeSplitTests: true # NEW: Enable split tests - - setup_for_tests_win: - name: Setup for tests (Windows) - runs-on: windows-latest - outputs: - integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }} - split_tests_matrix: ${{ steps.generate_tests_matrix.outputs.split_tests_matrix }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - uses: ./.github/actions/enumerate-tests - id: generate_tests_matrix - with: - includeIntegrations: true - includeSplitTests: true # NEW: Enable split tests -``` - -#### 2. Add split test jobs (NEW) - -```yaml - # NEW: Split tests for Linux - split_tests_lin: - uses: ./.github/workflows/run-tests.yml - name: Split Tests Linux - needs: [setup_for_tests_lin, build_packages] - if: ${{ fromJson(needs.setup_for_tests_lin.outputs.split_tests_matrix).include[0] != null }} - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.setup_for_tests_lin.outputs.split_tests_matrix) }} - with: - testShortName: "${{ matrix.projectName }}_${{ matrix.shortname }}" - testProjectPath: "${{ matrix.testProjectPath }}" - os: "ubuntu-latest" - testSessionTimeout: "${{ matrix.testSessionTimeout }}" - testHangTimeout: "${{ matrix.testHangTimeout }}" - extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true --filter-class ${{ matrix.fullClassName }}" - requiresNugets: ${{ matrix.requiresNugets }} - requiresTestSdk: ${{ matrix.requiresTestSdk }} - enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} - versionOverrideArg: ${{ inputs.versionOverrideArg }} - - # NEW: Split tests for macOS - split_tests_macos: - uses: ./.github/workflows/run-tests.yml - name: Split Tests macOS - needs: [setup_for_tests_macos, build_packages] - if: ${{ fromJson(needs.setup_for_tests_macos.outputs.split_tests_matrix).include[0] != null }} - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.setup_for_tests_macos.outputs.split_tests_matrix) }} - with: - testShortName: "${{ matrix.projectName }}_${{ matrix.shortname }}" - testProjectPath: "${{ matrix.testProjectPath }}" - os: "macos-latest" - testSessionTimeout: "${{ matrix.testSessionTimeout }}" - testHangTimeout: "${{ matrix.testHangTimeout }}" - extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true --filter-class ${{ matrix.fullClassName }}" - requiresNugets: ${{ matrix.requiresNugets }} - requiresTestSdk: ${{ matrix.requiresTestSdk }} - enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} - versionOverrideArg: ${{ inputs.versionOverrideArg }} - - # NEW: Split tests for Windows - split_tests_win: - uses: ./.github/workflows/run-tests.yml - name: Split Tests Windows - needs: [setup_for_tests_win, build_packages] - if: ${{ fromJson(needs.setup_for_tests_win.outputs.split_tests_matrix).include[0] != null }} - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.setup_for_tests_win.outputs.split_tests_matrix) }} - with: - testShortName: "${{ matrix.projectName }}_${{ matrix.shortname }}" - testProjectPath: "${{ matrix.testProjectPath }}" - os: "windows-latest" - testSessionTimeout: "${{ matrix.testSessionTimeout }}" - testHangTimeout: "${{ matrix.testHangTimeout }}" - extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true --filter-class ${{ matrix.fullClassName }}" - requiresNugets: ${{ matrix.requiresNugets }} - requiresTestSdk: ${{ matrix.requiresTestSdk }} - enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} - versionOverrideArg: ${{ inputs.versionOverrideArg }} -``` - -#### 3. REMOVE old templates_test_* jobs - -```yaml -# DELETE THESE (they'll use the new split_tests_* jobs instead): -# - templates_test_lin -# - templates_test_macos -# - templates_test_win -``` - -#### 4. Update results job dependencies - -```yaml - results: - if: ${{ always() && github.repository_owner == 'dotnet' }} - runs-on: ubuntu-latest - name: Final Test Results - needs: [ - endtoend_tests, - extension_tests_win, - integrations_test_lin, - integrations_test_macos, - integrations_test_win, - split_tests_lin, # NEW - split_tests_macos, # NEW - split_tests_win # NEW - ] - # ... rest of job unchanged ... -``` - -## Testing the Workflow Changes - -### Test 1: Dry Run with Empty Matrix - -Before enabling any split tests, verify the workflow handles empty matrices: - -1. Don't set `SplitTestsForCI=true` in any project -2. Push to a branch -3. Verify workflow runs successfully -4. Check that split_tests_* jobs are skipped (due to `if` condition) - -### Test 2: Enable for One Project - -1. Enable splitting for Aspire.Templates.Tests (already configured) -2. Push to a branch -3. Verify: - - 3 setup jobs run (one per OS) - - Each generates a matrix - - Split test jobs run in parallel - - Each test class runs separately - -### Test 3: Verify OS-Specific Matrices - -Check that each OS can have different matrices: - -1. Set a project to `RunOnGithubActionsLinux=true` but `RunOnGithubActionsWindows=false` -2. Verify Linux matrix includes it, Windows matrix doesn't -3. Verify Windows split_tests_win job is skipped or has fewer tests - -## Important Notes - -### Why Per-OS Setup Jobs? - -```yaml -# ❌ DON'T DO THIS - Single setup job -setup_for_tests: - runs-on: ubuntu-latest # Only Linux! - # This would only detect Linux tests - -# ✅ DO THIS - Per-OS setup jobs -setup_for_tests_lin: - runs-on: ubuntu-latest - -setup_for_tests_macos: - runs-on: macos-latest - -setup_for_tests_win: - runs-on: windows-latest -``` - -### Matrix Conditional - -The `if` condition prevents job failure when matrix is empty: - -```yaml -if: ${{ fromJson(needs.setup_for_tests_lin.outputs.split_tests_matrix).include[0] != null }} -``` - -This checks if the matrix has at least one entry. - -## Common Issues - -### Issue: "Invalid matrix" - -**Symptom**: Workflow fails with matrix parsing error -**Cause**: Malformed JSON from PowerShell script -**Fix**: Check `artifacts/test-matrices/split-tests-matrix.json` structure - -### Issue: Split tests not running - -**Symptom**: split_tests_* jobs are skipped -**Cause**: Empty matrix or missing `includeSplitTests: true` -**Fix**: Verify enumerate-tests action has correct inputs - -### Issue: Tests run on wrong OS - -**Symptom**: Linux tests running on Windows -**Cause**: Using wrong matrix output -**Fix**: Ensure each job uses the correct `needs.setup_for_tests_{os}.outputs` - -## Next Steps - -Proceed to [Step 4: Project Configuration](./STEP_04_PROJECT_CONFIG.md) \ No newline at end of file diff --git a/docs/test-splitting/STEP_03_MATRIX_GENERATOR_V3.md b/docs/test-splitting/STEP_03_MATRIX_GENERATOR_V3.md deleted file mode 100644 index 1bc65f3c128..00000000000 --- a/docs/test-splitting/STEP_03_MATRIX_GENERATOR_V3.md +++ /dev/null @@ -1,865 +0,0 @@ -# Step 3: Matrix Generator Implementation (v3 - Dual Mode Support) - -## Overview - -Enhanced PowerShell script that reads the auto-detected test lists and generates matrices for both collection-based and class-based splitting modes. - -## File: `eng/scripts/generate-test-matrix.ps1` - -### Complete Implementation - -```powershell -<# -.SYNOPSIS - Generates CI test matrices from auto-detected test enumeration files. - -.DESCRIPTION - This script reads .tests.list and .tests.metadata.json files and generates - a JSON matrix file for consumption by GitHub Actions or Azure DevOps. - - Automatically handles both modes: - - Collection-based: Entries like "collection:Name" and "uncollected:*" - - Class-based: Entries like "class:Full.Class.Name" - - The script is cross-platform and runs on Windows, Linux, and macOS. - -.PARAMETER TestListsDirectory - Directory containing .tests.list and .tests.metadata.json files. - Typically: artifacts/helix/ - -.PARAMETER OutputDirectory - Directory where the JSON matrix file will be written. - Typically: artifacts/test-matrices/ - -.PARAMETER BuildOs - Current operating system being built for (windows, linux, darwin). - Used for logging and debugging. - -.EXAMPLE - pwsh generate-test-matrix.ps1 -TestListsDirectory ./artifacts/helix -OutputDirectory ./artifacts/matrices -BuildOs linux - -.NOTES - Author: Aspire Team - Date: 2025-10-16 - Version: 3.0 (Auto-detection support) - Requires: PowerShell 7.0+ (cross-platform) -#> - -[CmdletBinding()] -param( - [Parameter(Mandatory=$true, HelpMessage="Directory containing test list files")] - [ValidateScript({Test-Path $_ -PathType Container})] - [string]$TestListsDirectory, - - [Parameter(Mandatory=$true, HelpMessage="Output directory for matrix JSON")] - [string]$OutputDirectory, - - [Parameter(Mandatory=$false, HelpMessage="Current OS: windows, linux, or darwin")] - [ValidateSet('windows', 'linux', 'darwin', '')] - [string]$BuildOs = '' -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version Latest - -#region Helper Functions - -function Write-Message { - <# - .SYNOPSIS - Writes a formatted message to the console. - #> - param( - [Parameter(Mandatory=$true)] - [AllowEmptyString()] - [string]$Message, - - [Parameter(Mandatory=$false)] - [ValidateSet('Info', 'Success', 'Warning', 'Error', 'Debug')] - [string]$Level = 'Info' - ) - - $prefix = switch ($Level) { - 'Success' { '✅' } - 'Warning' { '⚠️' } - 'Error' { '❌' } - 'Debug' { '🔍' } - default { 'ℹ️' } - } - - $color = switch ($Level) { - 'Success' { 'Green' } - 'Warning' { 'Yellow' } - 'Error' { 'Red' } - 'Debug' { 'Gray' } - default { 'Cyan' } - } - - Write-Host "$prefix $Message" -ForegroundColor $color -} - -function Get-TestListFiles { - <# - .SYNOPSIS - Finds all .tests.list files in the specified directory. - #> - param([string]$Directory) - - Get-ChildItem -Path $Directory -Filter "*.tests.list" -Recurse -ErrorAction SilentlyContinue -} - -function Read-TestMetadata { - <# - .SYNOPSIS - Reads and parses test metadata JSON file. - #> - param( - [string]$MetadataFile, - [string]$ProjectName - ) - - # Default metadata values - $defaults = @{ - projectName = $ProjectName - testClassNamesPrefix = $ProjectName - testProjectPath = "tests/$ProjectName/$ProjectName.csproj" - mode = 'class' - collections = '' - requiresNugets = 'false' - requiresTestSdk = 'false' - testSessionTimeout = '20m' - testHangTimeout = '10m' - uncollectedTestsSessionTimeout = '15m' - uncollectedTestsHangTimeout = '8m' - enablePlaywrightInstall = 'false' - } - - if (-not (Test-Path $MetadataFile)) { - Write-Message "No metadata file found for $ProjectName, using defaults" -Level Warning - return $defaults - } - - try { - $content = Get-Content $MetadataFile -Raw | ConvertFrom-Json - - # Merge with defaults (content overrides defaults) - foreach ($key in $content.PSObject.Properties.Name) { - $defaults[$key] = $content.$key - } - - return $defaults - } - catch { - Write-Message "Failed to parse metadata for ${ProjectName}: $_" -Level Warning - return $defaults - } -} - -function Get-CollectionFilterArg { - <# - .SYNOPSIS - Generates xUnit filter argument for a specific collection. - #> - param([string]$CollectionName) - - return "--filter-collection `"$CollectionName`"" -} - -function Get-UncollectedFilterArg { - <# - .SYNOPSIS - Generates xUnit filter argument to exclude all collections. - #> - param([string[]]$Collections) - - if ($Collections.Count -eq 0) { - # No collections to exclude - run all tests - return "" - } - - # Build filter to exclude all collections - $filters = $Collections | ForEach-Object { - "--filter-not-collection `"$_`"" - } - - return $filters -join ' ' -} - -function Get-ClassFilterArg { - <# - .SYNOPSIS - Generates xUnit filter argument for a specific test class. - #> - param([string]$ClassName) - - return "--filter-class `"$ClassName`"" -} - -function New-CollectionMatrixEntry { - <# - .SYNOPSIS - Creates a matrix entry for a collection. - #> - param( - [string]$CollectionName, - [string]$ProjectName, - [hashtable]$Metadata - ) - - $filterArg = Get-CollectionFilterArg -CollectionName $CollectionName - - [ordered]@{ - type = "collection" - name = $CollectionName - shortname = "Collection_$CollectionName" - projectName = $ProjectName - testProjectPath = $Metadata.testProjectPath - filterArg = $filterArg - requiresNugets = ($Metadata.requiresNugets -eq 'true') - requiresTestSdk = ($Metadata.requiresTestSdk -eq 'true') - testSessionTimeout = $Metadata.testSessionTimeout - testHangTimeout = $Metadata.testHangTimeout - enablePlaywrightInstall = ($Metadata.enablePlaywrightInstall -eq 'true') - } -} - -function New-UncollectedMatrixEntry { - <# - .SYNOPSIS - Creates a matrix entry for uncollected tests. - #> - param( - [string[]]$Collections, - [string]$ProjectName, - [hashtable]$Metadata - ) - - $filterArg = Get-UncollectedFilterArg -Collections $Collections - - # Use specific timeouts for uncollected tests (usually faster) - $sessionTimeout = if ($Metadata.uncollectedTestsSessionTimeout) { - $Metadata.uncollectedTestsSessionTimeout - } else { - $Metadata.testSessionTimeout - } - - $hangTimeout = if ($Metadata.uncollectedTestsHangTimeout) { - $Metadata.uncollectedTestsHangTimeout - } else { - $Metadata.testHangTimeout - } - - [ordered]@{ - type = "uncollected" - name = "UncollectedTests" - shortname = "Uncollected" - projectName = $ProjectName - testProjectPath = $Metadata.testProjectPath - filterArg = $filterArg - requiresNugets = ($Metadata.requiresNugets -eq 'true') - requiresTestSdk = ($Metadata.requiresTestSdk -eq 'true') - testSessionTimeout = $sessionTimeout - testHangTimeout = $hangTimeout - enablePlaywrightInstall = ($Metadata.enablePlaywrightInstall -eq 'true') - } -} - -function New-ClassMatrixEntry { - <# - .SYNOPSIS - Creates a matrix entry for a test class. - #> - param( - [string]$FullClassName, - [string]$ProjectName, - [hashtable]$Metadata - ) - - $prefix = $Metadata.testClassNamesPrefix - $shortname = $FullClassName - - # Strip prefix if present (e.g., "Aspire.Templates.Tests.MyClass" → "MyClass") - if ($prefix -and $FullClassName.StartsWith("$prefix.")) { - $shortname = $FullClassName.Substring($prefix.Length + 1) - } - - $filterArg = Get-ClassFilterArg -ClassName $FullClassName - - [ordered]@{ - type = "class" - fullClassName = $FullClassName - shortname = $shortname - projectName = $ProjectName - testProjectPath = $Metadata.testProjectPath - filterArg = $filterArg - requiresNugets = ($Metadata.requiresNugets -eq 'true') - requiresTestSdk = ($Metadata.requiresTestSdk -eq 'true') - testSessionTimeout = $Metadata.testSessionTimeout - testHangTimeout = $Metadata.testHangTimeout - enablePlaywrightInstall = ($Metadata.enablePlaywrightInstall -eq 'true') - } -} - -function Parse-TestListFile { - <# - .SYNOPSIS - Parses a .tests.list file and returns structured data. - #> - param([string]$FilePath) - - $lines = Get-Content $FilePath | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } - - $result = @{ - Mode = 'unknown' - Collections = [System.Collections.ArrayList]::new() - Classes = [System.Collections.ArrayList]::new() - HasUncollected = $false - } - - foreach ($line in $lines) { - if ($line -match '^collection:(.+)$') { - $result.Mode = 'collection' - [void]$result.Collections.Add($Matches[1].Trim()) - } - elseif ($line -match '^uncollected:') { - $result.HasUncollected = $true - } - elseif ($line -match '^class:(.+)$') { - $result.Mode = 'class' - [void]$result.Classes.Add($Matches[1].Trim()) - } - } - - return $result -} - -#endregion - -#region Main Script - -Write-Message "Starting matrix generation for BuildOs=$BuildOs" -Level Success -Write-Message "Test lists directory: $TestListsDirectory" -Write-Message "Output directory: $OutputDirectory" -Write-Message "" - -# Find all test list files -$listFiles = Get-TestListFiles -Directory $TestListsDirectory - -if ($listFiles.Count -eq 0) { - Write-Message "No test list files found in $TestListsDirectory" -Level Warning - Write-Message "Creating empty matrix file..." - - # Create empty matrix - $emptyMatrix = @{ include = @() } - $outputFile = Join-Path $OutputDirectory "split-tests-matrix.json" - - # Ensure output directory exists - if (-not (Test-Path $OutputDirectory)) { - New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null - } - - $emptyMatrix | ConvertTo-Json -Depth 10 -Compress | Set-Content -Path $outputFile -Encoding UTF8 - Write-Message "Created empty matrix: $outputFile" -Level Success - exit 0 -} - -Write-Message "Found $($listFiles.Count) test list file(s)" -Level Success -Write-Message "" - -# Process each test list file -$allEntries = [System.Collections.ArrayList]::new() -$stats = @{} - -foreach ($listFile in $listFiles) { - # Extract project name - $projectName = [System.IO.Path]::GetFileNameWithoutExtension($listFile.Name -replace '\.tests$', '') - - Write-Message "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -Level Info - Write-Message "Processing: $projectName" -Level Info - Write-Message "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -Level Info - - # Parse test list file - $parsed = Parse-TestListFile -FilePath $listFile.FullName - - if ($parsed.Mode -eq 'unknown') { - Write-Message " Unable to determine mode, skipping" -Level Warning - continue - } - - # Read metadata - $metadataFile = $listFile.FullName -replace '\.tests\.list$', '.tests.metadata.json' - $metadata = Read-TestMetadata -MetadataFile $metadataFile -ProjectName $projectName - - Write-Message " Mode: $($parsed.Mode)" -Level Info - - $projectStats = @{ - Mode = $parsed.Mode - Collections = 0 - Classes = 0 - Uncollected = 0 - } - - if ($parsed.Mode -eq 'collection') { - # Collection-based mode - Write-Message " Strategy: Collection-based splitting" -Level Success - Write-Message "" - - # Generate matrix entries for each collection - foreach ($collectionName in $parsed.Collections) { - Write-Message " ➕ Collection: $collectionName" -Level Debug - - $entry = New-CollectionMatrixEntry ` - -CollectionName $collectionName ` - -ProjectName $projectName ` - -Metadata $metadata - - [void]$allEntries.Add($entry) - $projectStats.Collections++ - } - - # Generate matrix entry for uncollected tests - if ($parsed.HasUncollected) { - Write-Message " ➕ Uncollected tests (all non-collection tests)" -Level Debug - - $entry = New-UncollectedMatrixEntry ` - -Collections $parsed.Collections.ToArray() ` - -ProjectName $projectName ` - -Metadata $metadata - - [void]$allEntries.Add($entry) - $projectStats.Uncollected = 1 - } - - $totalJobs = $projectStats.Collections + $projectStats.Uncollected - Write-Message "" - Write-Message " ✅ Generated $totalJobs job(s): $($projectStats.Collections) collection(s) + $($projectStats.Uncollected) uncollected" -Level Success - } - else { - # Class-based mode - Write-Message " Strategy: Class-based splitting" -Level Success - Write-Message "" - - # Generate matrix entries for each class - foreach ($className in $parsed.Classes) { - $shortName = $className -replace "^$($metadata.testClassNamesPrefix)\.", "" - Write-Message " ➕ Class: $shortName" -Level Debug - - $entry = New-ClassMatrixEntry ` - -FullClassName $className ` - -ProjectName $projectName ` - -Metadata $metadata - - [void]$allEntries.Add($entry) - $projectStats.Classes++ - } - - Write-Message "" - Write-Message " ✅ Generated $($projectStats.Classes) job(s): one per class" -Level Success - } - - $stats[$projectName] = $projectStats - Write-Message "" -} - -# Generate final matrix -$matrix = @{ - include = $allEntries.ToArray() -} - -# Write JSON file -$outputFile = Join-Path $OutputDirectory "split-tests-matrix.json" - -# Ensure output directory exists -if (-not (Test-Path $OutputDirectory)) { - New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null -} - -$jsonOutput = $matrix | ConvertTo-Json -Depth 10 -Compress -$jsonOutput | Set-Content -Path $outputFile -Encoding UTF8 -NoNewline - -# Summary -Write-Message "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -Level Info -Write-Message "Matrix Generation Summary" -Level Success -Write-Message "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -Level Info -Write-Message "" -Write-Message "Total Jobs: $($allEntries.Count)" -Level Success -Write-Message "Output File: $outputFile" -Level Success -Write-Message "" -Write-Message "Breakdown by Project:" -Level Info -Write-Message "" - -foreach ($proj in $stats.Keys | Sort-Object) { - $s = $stats[$proj] - - if ($s.Mode -eq 'collection') { - $summary = "$($s.Collections) collection(s) + $($s.Uncollected) uncollected" - Write-Message " 📦 $proj (collection mode): $summary" -Level Info - } - else { - $summary = "$($s.Classes) class(es)" - Write-Message " 📄 $proj (class mode): $summary" -Level Info - } -} - -Write-Message "" -Write-Message "Matrix generation complete! ✨" -Level Success - -#endregion -``` - -## Key Features - -### 1. Dual Mode Support - -```powershell -if ($parsed.Mode -eq 'collection') { - # Collection-based splitting - # Generate: collection entries + uncollected entry -} -else { - # Class-based splitting - # Generate: one entry per class -} -``` - -### 2. Auto-Detection via File Parsing - -```powershell -# Parse .tests.list file format -if ($line -match '^collection:(.+)$') { - $result.Mode = 'collection' - # ... -} -elseif ($line -match '^class:(.+)$') { - $result.Mode = 'class' - # ... -} -``` - -### 3. Unified Matrix Entry Creation - -Each mode has its own entry creator: -- `New-CollectionMatrixEntry`: For collection jobs -- `New-UncollectedMatrixEntry`: For uncollected catch-all -- `New-ClassMatrixEntry`: For individual test classes - -### 4. Rich Logging - -``` -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Processing: Aspire.Hosting.Tests -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - Mode: collection - Strategy: Collection-based splitting - - ➕ Collection: DatabaseTests - ➕ Collection: ContainerTests - ➕ Uncollected tests (all non-collection tests) - - ✅ Generated 3 job(s): 2 collection(s) + 1 uncollected -``` - -## Testing the Script - -### Test 1: Collection Mode - -Create test files: - -```bash -# artifacts/helix/Aspire.Hosting.Tests.tests.list -collection:DatabaseTests -collection:ContainerTests -uncollected:* -``` - -```json -// artifacts/helix/Aspire.Hosting.Tests.tests.metadata.json -{ - "projectName": "Aspire.Hosting.Tests", - "testProjectPath": "tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj", - "mode": "collection", - "collections": "DatabaseTests;ContainerTests", - "testSessionTimeout": "25m", - "testHangTimeout": "12m", - "uncollectedTestsSessionTimeout": "15m", - "uncollectedTestsHangTimeout": "8m" -} -``` - -Run script: - -```powershell -pwsh eng/scripts/generate-test-matrix.ps1 ` - -TestListsDirectory ./artifacts/helix ` - -OutputDirectory ./artifacts/test-matrices ` - -BuildOs linux -``` - -**Expected Console Output**: -``` -✅ Starting matrix generation for BuildOs=linux -ℹ️ Test lists directory: ./artifacts/helix -ℹ️ Output directory: ./artifacts/test-matrices - -✅ Found 1 test list file(s) - -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -ℹ️ Processing: Aspire.Hosting.Tests -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -ℹ️ Mode: collection -✅ Strategy: Collection-based splitting - -🔍 ➕ Collection: DatabaseTests -🔍 ➕ Collection: ContainerTests -🔍 ➕ Uncollected tests (all non-collection tests) - -✅ Generated 3 job(s): 2 collection(s) + 1 uncollected - -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -✅ Matrix Generation Summary -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -✅ Total Jobs: 3 -✅ Output File: ./artifacts/test-matrices/split-tests-matrix.json - -ℹ️ Breakdown by Project: - -ℹ️ 📦 Aspire.Hosting.Tests (collection mode): 2 collection(s) + 1 uncollected - -✅ Matrix generation complete! ✨ -``` - -**Expected JSON Output**: - -```json -{ - "include": [ - { - "type": "collection", - "name": "DatabaseTests", - "shortname": "Collection_DatabaseTests", - "projectName": "Aspire.Hosting.Tests", - "testProjectPath": "tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj", - "filterArg": "--filter-collection \"DatabaseTests\"", - "requiresNugets": false, - "requiresTestSdk": false, - "testSessionTimeout": "25m", - "testHangTimeout": "12m", - "enablePlaywrightInstall": false - }, - { - "type": "collection", - "name": "ContainerTests", - "shortname": "Collection_ContainerTests", - "projectName": "Aspire.Hosting.Tests", - "testProjectPath": "tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj", - "filterArg": "--filter-collection \"ContainerTests\"", - "requiresNugets": false, - "requiresTestSdk": false, - "testSessionTimeout": "25m", - "testHangTimeout": "12m", - "enablePlaywrightInstall": false - }, - { - "type": "uncollected", - "name": "UncollectedTests", - "shortname": "Uncollected", - "projectName": "Aspire.Hosting.Tests", - "testProjectPath": "tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj", - "filterArg": "--filter-not-collection \"DatabaseTests\" --filter-not-collection \"ContainerTests\"", - "requiresNugets": false, - "requiresTestSdk": false, - "testSessionTimeout": "15m", - "testHangTimeout": "8m", - "enablePlaywrightInstall": false - } - ] -} -``` - -### Test 2: Class Mode - -Create test files: - -```bash -# artifacts/helix/Aspire.Templates.Tests.tests.list -class:Aspire.Templates.Tests.BuildAndRunTemplateTests -class:Aspire.Templates.Tests.EmptyTemplateRunTests -class:Aspire.Templates.Tests.StarterTemplateRunTests -``` - -```json -// artifacts/helix/Aspire.Templates.Tests.tests.metadata.json -{ - "projectName": "Aspire.Templates.Tests", - "testClassNamesPrefix": "Aspire.Templates.Tests", - "testProjectPath": "tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj", - "mode": "class", - "collections": "", - "testSessionTimeout": "20m", - "testHangTimeout": "10m", - "requiresNugets": "true", - "requiresTestSdk": "true", - "enablePlaywrightInstall": "true" -} -``` - -Run script: - -```powershell -pwsh eng/scripts/generate-test-matrix.ps1 ` - -TestListsDirectory ./artifacts/helix ` - -OutputDirectory ./artifacts/test-matrices ` - -BuildOs linux -``` - -**Expected Console Output**: -``` -✅ Starting matrix generation for BuildOs=linux -... - -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -ℹ️ Processing: Aspire.Templates.Tests -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -ℹ️ Mode: class -✅ Strategy: Class-based splitting - -🔍 ➕ Class: BuildAndRunTemplateTests -🔍 ➕ Class: EmptyTemplateRunTests -🔍 ➕ Class: StarterTemplateRunTests - -✅ Generated 3 job(s): one per class - -... - -ℹ️ 📄 Aspire.Templates.Tests (class mode): 3 class(es) - -✅ Matrix generation complete! ✨ -``` - -**Expected JSON Output**: - -```json -{ - "include": [ - { - "type": "class", - "fullClassName": "Aspire.Templates.Tests.BuildAndRunTemplateTests", - "shortname": "BuildAndRunTemplateTests", - "projectName": "Aspire.Templates.Tests", - "testProjectPath": "tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj", - "filterArg": "--filter-class \"Aspire.Templates.Tests.BuildAndRunTemplateTests\"", - "requiresNugets": true, - "requiresTestSdk": true, - "testSessionTimeout": "20m", - "testHangTimeout": "10m", - "enablePlaywrightInstall": true - }, - { - "type": "class", - "fullClassName": "Aspire.Templates.Tests.EmptyTemplateRunTests", - "shortname": "EmptyTemplateRunTests", - ... - }, - { - "type": "class", - "fullClassName": "Aspire.Templates.Tests.StarterTemplateRunTests", - "shortname": "StarterTemplateRunTests", - ... - } - ] -} -``` - -### Test 3: Mixed Projects - -Create files for both projects above, then run: - -```powershell -pwsh eng/scripts/generate-test-matrix.ps1 ` - -TestListsDirectory ./artifacts/helix ` - -OutputDirectory ./artifacts/test-matrices ` - -BuildOs linux -``` - -**Expected**: 6 total jobs (3 from Hosting.Tests + 3 from Templates.Tests) - -**Console Summary**: -``` -ℹ️ Breakdown by Project: - -ℹ️ 📦 Aspire.Hosting.Tests (collection mode): 2 collection(s) + 1 uncollected -ℹ️ 📄 Aspire.Templates.Tests (class mode): 3 class(es) -``` - -## Validation - -### Verify Matrix Structure - -```powershell -# Load matrix -$matrix = Get-Content ./artifacts/test-matrices/split-tests-matrix.json | ConvertFrom-Json - -# Check entry count -$matrix.include.Count - -# Verify all entries have required fields -$matrix.include | ForEach-Object { - $required = @('type', 'shortname', 'projectName', 'testProjectPath', 'filterArg') - foreach ($field in $required) { - if (-not $_.$field) { - Write-Error "Missing field: $field in entry: $($_.shortname)" - } - } -} - -# Check filter arguments -$matrix.include | Select-Object shortname, filterArg | Format-Table - -# Group by type -$matrix.include | Group-Object -Property type | Select-Object Name, Count -``` - -### Verify Filter Arguments Work - -```powershell -# Test a collection filter -dotnet test YourTests.dll -- --filter-collection "DatabaseTests" - -# Test a class filter -dotnet test YourTests.dll -- --filter-class "Aspire.Templates.Tests.Test1" - -# Test uncollected filter -dotnet test YourTests.dll -- --filter-not-collection "DatabaseTests" --filter-not-collection "ContainerTests" -``` - -## Common Issues - -### Issue 1: "Mode is unknown" - -**Symptom**: Script skips project with "Unable to determine mode" -**Cause**: .tests.list file has unexpected format -**Fix**: Check file format - should have `collection:` or `class:` prefixes - -### Issue 2: Invalid JSON - -**Symptom**: GitHub Actions can't parse matrix -**Cause**: Special characters in names -**Fix**: Script escapes quotes automatically, but verify with `jq` - -```bash -cat split-tests-matrix.json | jq empty -# Should exit with code 0 if valid -``` - -### Issue 3: Empty filterArg for uncollected - -**Symptom**: Uncollected job has empty filter -**Cause**: No collections to exclude -**Fix**: This is OK - empty filter runs all tests - -## Next Steps - -The matrix is now generated! GitHub Actions workflow already consumes it (no changes needed from v1). - -Proceed to [Step 4: Project Configuration (v3)](./STEP_04_PROJECT_CONFIG_V3.md) \ No newline at end of file diff --git a/docs/test-splitting/STEP_04_PROJECT_CONFIG.md b/docs/test-splitting/STEP_04_PROJECT_CONFIG.md deleted file mode 100644 index 3cb4508aac8..00000000000 --- a/docs/test-splitting/STEP_04_PROJECT_CONFIG.md +++ /dev/null @@ -1,230 +0,0 @@ -# Step 4: Project Configuration - -## Overview - -Configure test projects to use the new unified splitting mechanism. This step shows how to migrate existing projects and enable new ones. - -## Configuration Properties - -### Required Properties (for splitting) - -```xml - -true - - -Aspire.Hosting.Tests -``` - -### Optional Properties - -```xml - -QuickTest1;QuickTest2 - - -25m -12m - - -true - - -true - - -true -``` - -## Migration: Aspire.Templates.Tests - -### Before (Custom Implementation) - -```xml - - - $(DefaultTargetFramework) - - true - true - - xunit.runner.json - $(TestArchiveTestsDirForTemplateTests) - - - true - Aspire.Templates.Tests - - $(NoWarn);xUnit1051 - true - - - - - - - - - - - - -``` - -### After (Unified Mechanism) - -```xml - - - $(DefaultTargetFramework) - - true - true - - xunit.runner.json - - - true - Aspire.Templates.Tests - - - true - true - true - - - 20m - 12m - - $(NoWarn);xUnit1051 - true - - - - - - - - - - - - -``` - -### Changes Summary - -- ✅ Replace `ExtractTestClassNamesForHelix` with `SplitTestsForCI` -- ✅ Keep `TestClassNamesPrefix` (same property name) -- ✅ Add `RequiresNugetsForSplitTests=true` -- ✅ Add `RequiresTestSdkForSplitTests=true` -- ✅ Add `EnablePlaywrightInstallForSplitTests=true` -- ✅ Add timeout configurations -- ✅ Remove `TestArchiveTestsDir` override (use default) - -## New Project: Aspire.Hosting.Tests - -### Complete Configuration - -```xml - - - $(DefaultTargetFramework) - - - true - Aspire.Hosting.Tests - - - 25m - 15m - - - false - false - false - - - - - - - -``` - -## OS-Specific Opt-In/Out - -### Example: Linux-Only Splitting - -Some projects may only need splitting on Linux (e.g., Docker tests): - -```xml - - - true - Aspire.Docker.Tests - - - true - false - true - -``` - -This creates: -- **Linux**: Split into multiple jobs (one per class) -- **Windows**: Single job (no splitting) -- **macOS**: Doesn't run at all - -## Projects to Enable Splitting - -### High Priority (Long-Running) - -1. **Aspire.Templates.Tests** ✅ (Already has splitting, migrate to new mechanism) - - Currently: ~15 test classes - - Timeout: 20m - - Needs: Packages, SDK, Playwright - -2. **Aspire.Hosting.Tests** 🎯 (Primary target) - - Estimated: 50+ test classes - - Timeout: 25m - - Needs: None (regular integration test) - -3. **Aspire.Hosting.*.Tests** (if long-running) - - Aspire.Hosting.Azure.Tests - - Aspire.Hosting.Postgres.Tests - - etc. - -### Medium Priority - -4. Other integration tests if they exceed 15 minutes - -### Low Priority - -- Unit tests (usually fast enough) -- Tests with < 5 test classes (overhead not worth it) - -## Configuration Decision Tree - -``` -Is the test project slow (>15 minutes)? -│ -├─ NO → Don't enable splitting -│ (Keep as regular test) -│ -└─ YES → Does it have >5 test classes? - │ - ├─ NO → Don't enable splitting - │ (Won't benefit from parallelization) - │ - └─ YES → Enable splitting! - │ - ├─ Set SplitTestsForCI=true - ├─ Set TestClassNamesPrefix - ├─ Set custom timeouts if needed - └─ Set requirements (packages/SDK/etc.) -``` - -## Validation Checklist - -Before \ No newline at end of file diff --git a/docs/test-splitting/STEP_04_PROJECT_CONFIG_V2.md b/docs/test-splitting/STEP_04_PROJECT_CONFIG_V2.md deleted file mode 100644 index 9e87498a599..00000000000 --- a/docs/test-splitting/STEP_04_PROJECT_CONFIG_V2.md +++ /dev/null @@ -1,490 +0,0 @@ -# Step 4: Project Configuration (v2 - Collection Support) - -## Overview - -Configure test projects to use collection-based splitting with examples showing how to optimize test execution. - -## Configuration Properties - -### Required Properties - -```xml - -true - - -Aspire.Hosting.Tests -``` - -### Optional Properties (v2 Enhancements) - -```xml - -QuickTests;FastTests - - -25m -12m - - -15m -8m - - -false -false -false -``` - -## Example 1: Aspire.Hosting.Tests (NEW - Collections) - -### Project File Configuration - -```xml - - - $(DefaultTargetFramework) - - - true - Aspire.Hosting.Tests - - - 30m - 15m - - - 15m - 8m - - - false - false - - - - -``` - -### Test Class Organization - -```csharp -using Xunit; - -namespace Aspire.Hosting.Tests; - -// Slow database tests - group together -[Collection("DatabaseIntegration")] -public class PostgresLifecycleTests -{ - [Fact] - public async Task CanStartPostgresContainer() - { - // 2-3 minutes per test - } - - [Fact] - public async Task CanConnectToPostgres() - { - // 2-3 minutes per test - } -} - -[Collection("DatabaseIntegration")] -public class SqlServerLifecycleTests -{ - [Fact] - public async Task CanStartSqlServerContainer() - { - // 2-3 minutes per test - } -} - -// Slow container tests - separate group -[Collection("ContainerLifecycle")] -public class DockerContainerTests -{ - [Fact] - public async Task CanStartGenericContainer() - { - // 2-3 minutes per test - } - - [Fact] - public async Task CanStopContainer() - { - // 2 minutes per test - } -} - -[Collection("ContainerLifecycle")] -public class ContainerNetworkingTests -{ - [Fact] - public async Task ContainersCanCommunicate() - { - // 3 minutes per test - } -} - -// Fast unit tests - NO collection attribute -public class ConfigurationTests -{ - [Fact] - public void CanParseConfiguration() - { - // < 1 second - } - - [Fact] - public void CanValidateSettings() - { - // < 1 second - } -} - -public class UtilityTests -{ - [Fact] - public void HelperMethodWorks() - { - // < 1 second - } -} -``` - -### Expected CI Behavior - -**Before** (1 job): -``` -Aspire.Hosting.Tests: 55 minutes -``` - -**After** (3 jobs running in parallel): -``` -Collection_DatabaseIntegration: ~20 minutes (Postgres + SqlServer tests) -Collection_ContainerLifecycle: ~15 minutes (Docker + Networking tests) -UncollectedTests: ~5 minutes (Config + Utility tests) -``` - -**Total CI Time**: ~20 minutes (60% reduction!) - -## Example 2: Aspire.Templates.Tests (MIGRATED) - -### Before (v1 - Class-based splitting) - -```xml - - - $(DefaultTargetFramework) - - - true - Aspire.Templates.Tests - - true - true - - -``` - -### After (v2 - Collection-based splitting) - -```xml - - - $(DefaultTargetFramework) - - - true - Aspire.Templates.Tests - - - true - true - true - - - 25m - 15m - - - 15m - 10m - - true - true - - -``` - -### Test Class Organization Strategy - -```csharp -using Xunit; - -namespace Aspire.Templates.Tests; - -// Slow Playwright tests for starter template - group together -[Collection("StarterTemplateWithPlaywright")] -public class StarterTemplateProjectNamesTests -{ - // Each test: 3-5 minutes (Playwright browser automation) -} - -[Collection("StarterTemplateWithPlaywright")] -public class StarterTemplateRunTests -{ - // Each test: 3-5 minutes -} - -// Slow Playwright tests for basic template - separate group -[Collection("BasicTemplateWithPlaywright")] -public class BuildAndRunTemplateTests -{ - // Each test: 3-5 minutes -} - -// Build-only tests (no Playwright) - NO collection -public class NewUpAndBuildStandaloneTemplateTests -{ - // Each test: 1-2 minutes (just dotnet build) -} - -public class TemplateManifestTests -{ - // Each test: < 1 minute (metadata tests) -} -``` - -**Result**: 3 jobs -1. Collection_StarterTemplateWithPlaywright (~15 min) -2. Collection_BasicTemplateWithPlaywright (~12 min) -3. UncollectedTests (~5 min) - -## Example 3: Simple Project (No Collections) - -### When NOT to Use Collections - -```xml - - - $(DefaultTargetFramework) - - - true - Aspire.MySqlConnector.Tests - - - 15m - - -``` - -```csharp -// All test classes without [Collection] attribute -public class ConnectionTests { } -public class QueryTests { } -public class TransactionTests { } -``` - -**Result**: 1 job (UncollectedTests) running all tests - -**When to use this**: -- Project has < 15 minute total runtime -- All tests are similar speed -- No benefit from parallelization - -## Example 4: Excluding Collections - -### Scenario: Some Collections Shouldn't Split - -```xml - - true - Aspire.Hosting.Tests - - - QuickIntegrationTests;FastSmokeTests - -``` - -```csharp -[Collection("SlowDatabaseTests")] -public class SlowTests { } // Gets own job - -[Collection("QuickIntegrationTests")] // Excluded from splitting -public class QuickTests { } // Runs in UncollectedTests - -public class OtherTests { } // Runs in UncollectedTests -``` - -**Result**: 2 jobs -1. Collection_SlowDatabaseTests -2. UncollectedTests (includes QuickIntegrationTests + OtherTests) - -## Decision Matrix: Should You Use Collections? - -### ✅ Use Collections When: - -| Scenario | Example | -|----------|---------| -| **Shared expensive setup** | Database containers that multiple test classes use | -| **Long-running integration tests** | Tests that take 2+ minutes each | -| **Logical test grouping** | All Azure tests, all Docker tests, etc. | -| **Similar resource needs** | Tests that all need Playwright, or all need databases | - -### ❌ Don't Use Collections When: - -| Scenario | Reason | -|----------|--------| -| **Fast unit tests** | Overhead isn't worth it; let them run together | -| **< 5 total test classes** | Not enough parallelization benefit | -| **Tests need isolation** | Collections share fixtures which may cause conflicts | -| **Total runtime < 15 min** | Single job is fast enough | - -## Migration Checklist - -### For Each Long-Running Project: - -- [ ] Analyze test suite duration -- [ ] Identify slow test groups (> 10 min combined) -- [ ] Add `[Collection("GroupName")]` to slow test classes -- [ ] Keep fast tests without collection attribute -- [ ] Update .csproj with split configuration -- [ ] Set appropriate timeouts -- [ ] Test locally first -- [ ] Monitor CI times after merge - -## Best Practices - -### 1. Collection Naming - -```csharp -// ✅ Good: Descriptive, indicates purpose -[Collection("DatabaseIntegrationTests")] -[Collection("ContainerLifecycleTests")] -[Collection("PlaywrightAutomationTests")] - -// ❌ Bad: Too vague or too specific -[Collection("Tests")] // Too vague -[Collection("PostgresTest1")] // Too specific -``` - -### 2. Collection Size - -```csharp -// ✅ Good: Multiple related test classes in one collection -[Collection("DatabaseTests")] -public class PostgresTests { /* 10 tests */ } - -[Collection("DatabaseTests")] -public class MySqlTests { /* 8 tests */ } - -[Collection("DatabaseTests")] -public class SqlServerTests { /* 12 tests */ } -// Total: 30 tests, ~20 minutes - good parallelization unit - -// ❌ Bad: One test class per collection -[Collection("PostgresTests")] -public class PostgresTests { /* 10 tests */ } - -[Collection("MySqlTests")] -public class MySqlTests { /* 8 tests */ } -// Too granular, overhead not worth it -``` - -### 3. Timeout Configuration - -```xml - -30m - - -10m -``` - -### 4. Test Isolation - -```csharp -// ✅ Good: Tests in same collection can share fixtures -[Collection("DatabaseTests")] -public class PostgresTests : IClassFixture -{ - // Fixture is shared across collection -} - -[Collection("DatabaseTests")] -public class MySqlTests : IClassFixture -{ - // Same fixture instance - efficient! -} - -// ❌ Bad: Tests that MUST be isolated shouldn't share collection -[Collection("IsolatedTests")] // Don't do this -public class Test1 { /* Modifies global state */ } - -[Collection("IsolatedTests")] // Will conflict with Test1 -public class Test2 { /* Also modifies global state */ } -``` - -## Validation After Configuration - -### 1. Build Locally - -```bash -dotnet build tests/YourProject.Tests/YourProject.Tests.csproj \ - /t:Build;ExtractTestClassNames \ - -p:PrepareForHelix=true \ - -p:SplitTestsForCI=true \ - -p:TestClassNamesPrefix=YourProject.Tests -``` - -### 2. Check Generated Files - -```bash -# Should see: -ls artifacts/helix/YourProject.Tests.tests.list -ls artifacts/helix/YourProject.Tests.tests.metadata.json - -# Content should be: -cat artifacts/helix/YourProject.Tests.tests.list -# collection:YourCollection1 -# collection:YourCollection2 -# uncollected:* -``` - -### 3. Generate Matrix - -```bash -pwsh eng/scripts/generate-test-matrix.ps1 \ - -TestListsDirectory ./artifacts/helix \ - -OutputDirectory ./artifacts/test-matrices \ - -BuildOs linux -``` - -### 4. Verify Matrix - -```bash -cat artifacts/test-matrices/split-tests-matrix.json | jq '.include[] | {name, filterArg}' -``` - -**Expected output**: -```json -{ - "name": "YourCollection1", - "filterArg": "--filter-collection \"YourCollection1\"" -} -{ - "name": "YourCollection2", - "filterArg": "--filter-collection \"YourCollection2\"" -} -{ - "name": "UncollectedTests", - "filterArg": "--filter-not-collection \"YourCollection1\" --filter-not-collection \"YourCollection2\"" -} -``` - -## Next Steps - -Proceed to [Step 5: Testing & Validation (v2)](./STEP_05_TESTING_V2.md) \ No newline at end of file diff --git a/docs/test-splitting/STEP_04_PROJECT_CONFIG_V3.md b/docs/test-splitting/STEP_04_PROJECT_CONFIG_V3.md deleted file mode 100644 index f1093378513..00000000000 --- a/docs/test-splitting/STEP_04_PROJECT_CONFIG_V3.md +++ /dev/null @@ -1,316 +0,0 @@ -# Step 4: Project Configuration (v3 - Simplified) - -## Overview - -With v3's auto-detection, project configuration is minimal. Just set two properties and the system automatically detects whether to use collection or class-based splitting. - -## Minimal Configuration - -### Required Properties (Only 2!) - -```xml - - - true - - - YourProject.Tests - -``` - -That's it! The system auto-detects collections and chooses the optimal strategy. - -## Configuration Examples - -### Example 1: Aspire.Hosting.Tests (NEW - With Collections) - -#### Step 1: Configure Project - -```xml name=tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj - - - $(DefaultTargetFramework) - - - true - Aspire.Hosting.Tests - - - 30m - 15m - 15m - 8m - - - - -``` - -#### Step 2: Add Collections to Test Classes - -```csharp -using Xunit; - -namespace Aspire.Hosting.Tests; - -// Group slow database tests together -[Collection("DatabaseIntegration")] -public class PostgresLifecycleTests -{ - [Fact] - public async Task CanStartPostgresContainer() - { - // Test implementation - } -} - -[Collection("DatabaseIntegration")] -public class MySqlLifecycleTests -{ - [Fact] - public async Task CanStartMySqlContainer() - { - // Test implementation - } -} - -// Group container tests together -[Collection("ContainerLifecycle")] -public class DockerContainerTests -{ - [Fact] - public async Task CanStartGenericContainer() - { - // Test implementation - } -} - -// Fast tests - NO collection attribute -public class ConfigurationTests -{ - [Fact] - public void CanParseConfiguration() - { - // Fast unit test - } -} - -public class UtilityTests -{ - [Fact] - public void HelperMethodWorks() - { - // Fast unit test - } -} -``` - -#### Result - -**Auto-detected mode**: Collection (2 collections found) -**CI Jobs**: 3 -- `Collection_DatabaseIntegration` (Postgres + MySQL tests) -- `Collection_ContainerLifecycle` (Docker tests) -- `Uncollected` (Configuration + Utility tests) - -**Before**: 1 job, 60 minutes -**After**: 3 parallel jobs, ~25 minutes (58% reduction) - -### Example 2: Aspire.Templates.Tests (MIGRATE from Old System) - -#### Before (Custom Mechanism) - -```xml - - - true - Aspire.Templates.Tests - $(TestArchiveTestsDirForTemplateTests) - -``` - -#### After (Unified v3 Mechanism) - -```xml name=tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj - - - $(DefaultTargetFramework) - - - true - Aspire.Templates.Tests - - - true - true - true - - - 20m - 12m - - - true - true - xunit.runner.json - $(NoWarn);xUnit1051 - - - - - - - - - - - -``` - -#### Test Classes (No Changes Needed) - -```csharp -// Existing test classes without [Collection] attributes -public class BuildAndRunTemplateTests { } -public class EmptyTemplateRunTests { } -public class StarterTemplateRunTests { } -// ... etc -``` - -#### Result - -**Auto-detected mode**: Class (no collections found) -**CI Jobs**: 12 (one per test class) -**Behavior**: Identical to old system, but using unified infrastructure - -### Example 3: Simple Project (No Splitting Needed) - -```xml name=tests/Aspire.MySqlConnector.Tests/Aspire.MySqlConnector.Tests.csproj - - - $(DefaultTargetFramework) - -``` - -**Result**: 1 job (existing behavior, no splitting) - -## Optional Configuration Properties - -### Timeouts - -```xml - -20m -10m - - -15m -8m -``` - -### Test Requirements - -```xml - -true - - -true - - -true -``` - -### Collection Management - -```xml - -FastTests;QuickTests -``` - -These collections will run in the `Uncollected` job instead. - -## Decision Guide - -### Should I Enable Splitting? - -``` -Is total test time > 15 minutes? -│ -├─ NO → Don't enable SplitTestsOnCI -│ Overhead not worth it -│ -└─ YES → Enable SplitTestsOnCI=true - │ - Do you have logical test groups? - │ - ├─ YES → Add [Collection] attributes - │ System auto-detects: Collection mode - │ Result: Fewer jobs, better parallelization - │ - └─ NO → Leave tests as-is - System auto-detects: Class mode - Result: One job per class -``` - -### Collection Size Guidelines - -**Good Collection** (15-30 minutes): -```csharp -[Collection("DatabaseTests")] -public class PostgresTests { /* 20 tests, 8 min */ } - -[Collection("DatabaseTests")] -public class MySqlTests { /* 15 tests, 7 min */ } - -[Collection("DatabaseTests")] -public class SqlServerTests { /* 25 tests, 10 min */ } - -// Total: ~25 minutes - ideal for one job -``` - -**Too Small** (< 5 minutes): -```csharp -[Collection("QuickTest")] -public class OneTest { /* 2 tests, 1 min */ } - -// Don't create collections for fast tests -// Let them run in the uncollected job -``` - -**Too Large** (> 45 minutes): -```csharp -[Collection("AllDatabaseTests")] -public class Test1 { /* 100 tests */ } -public class Test2 { /* 100 tests */ } -// ... - -// Split into multiple smaller collections instead -``` - -## Migration Checklist - -### For Each Long-Running Project: - -- [ ] Measure current test duration -- [ ] If > 15 min, enable `SplitTestsOnCI=true` -- [ ] Set `TestClassNamesPrefix` -- [ ] (Optional) Add `[Collection]` to slow test groups -- [ ] Test locally (see Step 5) -- [ ] Create PR -- [ ] Monitor CI times after merge - -### Specific Migration: Aspire.Templates.Tests - -- [ ] Replace `ExtractTestClassNamesForHelix` with `SplitTestsOnCI` -- [ ] Keep `TestClassNamesPrefix` (same name) -- [ ] Add `RequiresNugetsForSplitTests=true` -- [ ] Add `RequiresTestSdkForSplitTests=true` -- [ ] Add `EnablePlaywrightInstallForSplitTests=true` -- [ ] Remove `TestArchiveTestsDir` override -- [ ] Test locally -- [ ] Verify same number of jobs in CI - -## Next Steps - -Proceed to [Step 5: Testing & Validation](./STEP_05_TESTING_V3.md) \ No newline at end of file diff --git a/docs/test-splitting/STEP_05_TESTING_V3.md b/docs/test-splitting/STEP_05_TESTING_V3.md deleted file mode 100644 index 92b3e54e3a0..00000000000 --- a/docs/test-splitting/STEP_05_TESTING_V3.md +++ /dev/null @@ -1,373 +0,0 @@ -# Step 5: Testing & Validation Guide - -## Overview - -This guide provides step-by-step instructions for testing the implementation locally before pushing to CI. - -## Prerequisites - -- PowerShell 7.0+ installed -- .NET SDK matching `global.json` -- Aspire repository cloned locally - -## Phase 1: Test PowerShell Scripts in Isolation - -### Test 1: Discovery Helper Script - -```powershell -# Create mock test output -$mockOutput = @( - "Collection: DatabaseTests", - " Aspire.Hosting.Tests.PostgresTests.CanStartContainer", - " Aspire.Hosting.Tests.PostgresTests.CanConnect", - "Collection: ContainerTests", - " Aspire.Hosting.Tests.DockerTests.CanStartContainer", - "Aspire.Hosting.Tests.QuickTests.FastTest1" -) - -# Test the script -pwsh eng/scripts/extract-test-metadata.ps1 ` - -TestAssemblyOutput $mockOutput ` - -TestClassNamesPrefix "Aspire.Hosting.Tests" ` - -OutputListFile "./test-output.list" -``` - -**Expected Output File**: -``` -collection:ContainerTests -collection:DatabaseTests -uncollected:* -``` - -**Validation**: -- [ ] Script runs without errors -- [ ] Output file created -- [ ] Contains 3 lines (2 collections + uncollected) -- [ ] Collections are sorted alphabetically - -### Test 2: Matrix Generator Script - -```powershell -# Create test files -mkdir -p artifacts/helix - -# Create .tests.list -@" -collection:DatabaseTests -collection:ContainerTests -uncollected:* -"@ | Out-File -FilePath artifacts/helix/TestProject.tests.list -Encoding UTF8 - -# Create .tests.metadata.json -@" -{ - "projectName": "TestProject", - "testProjectPath": "tests/TestProject/TestProject.csproj", - "mode": "collection", - "collections": "DatabaseTests;ContainerTests", - "testSessionTimeout": "20m", - "testHangTimeout": "10m" -} -"@ | Out-File -FilePath artifacts/helix/TestProject.tests.metadata.json -Encoding UTF8 - -# Run generator -pwsh eng/scripts/generate-test-matrix.ps1 ` - -TestListsDirectory ./artifacts/helix ` - -OutputDirectory ./artifacts/test-matrices ` - -BuildOs linux -``` - -**Expected Output**: -- [ ] Matrix JSON file created -- [ ] Contains 3 entries (2 collections + 1 uncollected) -- [ ] Each entry has `type`, `name`, `shortname`, `filterArg`, etc. -- [ ] Filter args are correct: - - `--filter-collection "DatabaseTests"` - - `--filter-collection "ContainerTests"` - - `--filter-not-collection "DatabaseTests" --filter-not-collection "ContainerTests"` - -**Validate JSON**: -```powershell -# Check JSON is valid -$matrix = Get-Content ./artifacts/test-matrices/split-tests-matrix.json | ConvertFrom-Json -$matrix.include.Count # Should be 3 - -# Or use jq -jq '.include | length' ./artifacts/test-matrices/split-tests-matrix.json -# Should output: 3 -``` - -## Phase 2: Test MSBuild Integration - -### Test 1: Build Test Project with Splitting Enabled - -Choose a test project to experiment with (or create a dummy one): - -```bash -# Build with splitting enabled -dotnet build tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj \ - /t:Build;ExtractTestClassNames \ - -p:PrepareForHelix=true \ - -p:SplitTestsOnCI=true \ - -p:TestClassNamesPrefix=Aspire.Templates.Tests \ - -p:InstallBrowsersForPlaywright=false \ - /bl:build.binlog -``` - -**Expected Output**: -``` -[Aspire.Templates.Tests] Starting test metadata extraction... -[Aspire.Templates.Tests] Running discovery helper... -ℹ️ Parsing test assembly output... -✅ Detection Results: -ℹ️ Mode: class (or "collection" if you added [Collection] attributes) -... -[Aspire.Templates.Tests] ✅ Test metadata extraction complete! -``` - -**Validation**: -- [ ] Build succeeds -- [ ] Files created in `artifacts/helix/`: - - [ ] `Aspire.Templates.Tests.tests.list` - - [ ] `Aspire.Templates.Tests.tests.metadata.json` -- [ ] Binlog shows ExtractTestClassNames target executed -- [ ] No errors in console output - -### Test 2: Verify Generated Files - -```bash -# Check .tests.list -cat artifacts/helix/Aspire.Templates.Tests.tests.list - -# Check metadata -cat artifacts/helix/Aspire.Templates.Tests.tests.metadata.json | jq . - -# Verify mode -cat artifacts/helix/Aspire.Templates.Tests.tests.metadata.json | jq -r .mode -# Should output: "class" or "collection" -``` - -### Test 3: Generate Matrix - -```bash -# Run the full GetTestProjects.proj -dotnet build tests/Shared/GetTestProjects.proj \ - /p:TestsListOutputPath=$PWD/artifacts/TestsForGithubActions.list \ - /p:TestMatrixOutputPath=$PWD/artifacts/test-matrices/ \ - /p:ContinuousIntegrationBuild=true \ - /bl:get-test-projects.binlog -``` - -**Validation**: -- [ ] `artifacts/TestsForGithubActions.list` created (regular tests) -- [ ] `artifacts/TestsForGithubActions.list.split-projects` created (split tests) -- [ ] `artifacts/test-matrices/split-tests-matrix.json` created -- [ ] Matrix JSON is valid - -```bash -# Validate -jq . artifacts/test-matrices/split-tests-matrix.json -``` - -## Phase 3: Test with Real Project - -### Option A: Test with Aspire.Templates.Tests (No Collections) - -```bash -# 1. Update .csproj (already has splitting, just verify) -# 2. Build -./build.sh -restore -build -projects tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj - -# 3. Extract metadata -dotnet build tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj \ - /t:Build;ExtractTestClassNames \ - -p:PrepareForHelix=true \ - -p:SplitTestsOnCI=true \ - -p:InstallBrowsersForPlaywright=false - -# 4. Check mode -cat artifacts/helix/Aspire.Templates.Tests.tests.metadata.json | jq -r .mode -# Expected: "class" - -# 5. Count entries -cat artifacts/helix/Aspire.Templates.Tests.tests.list | wc -l -# Expected: ~12 (one per test class) -``` - -### Option B: Test with Aspire.Hosting.Tests (Add Collections) - -```bash -# 1. Add [Collection] attributes to some test classes -# Edit: tests/Aspire.Hosting.Tests/SomeTests.cs - -# 2. Enable splitting in .csproj -# Add: -# true -# Aspire.Hosting.Tests - -# 3. Build -./build.sh -restore -build -projects tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj - -# 4. Extract metadata -dotnet build tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj \ - /t:Build;ExtractTestClassNames \ - -p:PrepareForHelix=true \ - -p:SplitTestsOnCI=true - -# 5. Check mode -cat artifacts/helix/Aspire.Hosting.Tests.tests.metadata.json | jq -r .mode -# Expected: "collection" - -# 6. Check collections -cat artifacts/helix/Aspire.Hosting.Tests.tests.list -# Expected: -# collection:YourCollectionName -# collection:AnotherCollection -# uncollected:* -``` - -## Phase 4: Test Full Workflow Locally - -### Simulate GitHub Actions Enumerate Step - -```bash -# Run the enumerate-tests action logic locally -dotnet build tests/Shared/GetTestProjects.proj \ - /p:TestsListOutputPath=$PWD/artifacts/TestsForGithubActions.list \ - /p:TestMatrixOutputPath=$PWD/artifacts/test-matrices/ \ - /p:ContinuousIntegrationBuild=true - -# Check split projects -cat artifacts/TestsForGithubActions.list.split-projects -# Should list: Templates or Hosting (whichever has SplitTestsOnCI=true) - -# Build each split project -while read project; do - echo "Building $project..." - dotnet build tests/Aspire.$project.Tests/Aspire.$project.Tests.csproj \ - /t:Build;ExtractTestClassNames \ - -p:PrepareForHelix=true \ - -p:SplitTestsOnCI=true \ - -p:InstallBrowsersForPlaywright=false -done < artifacts/TestsForGithubActions.list.split-projects - -# Generate matrix -pwsh eng/scripts/generate-test-matrix.ps1 \ - -TestListsDirectory ./artifacts/helix \ - -OutputDirectory ./artifacts/test-matrices \ - -BuildOs linux - -# Verify matrix -jq '.include[] | {shortname, filterArg}' artifacts/test-matrices/split-tests-matrix.json -``` - -## Phase 5: Verify Filter Arguments Work - -### Test Collection Filter - -```bash -# Run tests with collection filter -dotnet test artifacts/bin/Aspire.Hosting.Tests/Debug/net9.0/Aspire.Hosting.Tests.dll \ - -- --filter-collection "DatabaseTests" - -# Should only run tests in DatabaseTests collection -``` - -### Test Class Filter - -```bash -# Run tests with class filter -dotnet test artifacts/bin/Aspire.Templates.Tests/Debug/net9.0/Aspire.Templates.Tests.dll \ - -- --filter-class "Aspire.Templates.Tests.BuildAndRunTemplateTests" - -# Should only run tests in that class -``` - -### Test Uncollected Filter - -```bash -# Run tests NOT in collections -dotnet test artifacts/bin/Aspire.Hosting.Tests/Debug/net9.0/Aspire.Hosting.Tests.dll \ - -- --filter-not-collection "DatabaseTests" --filter-not-collection "ContainerTests" - -# Should only run tests without [Collection] attributes -``` - -## Validation Checklist - -### PowerShell Scripts -- [ ] `extract-test-metadata.ps1` runs without errors -- [ ] `extract-test-metadata.ps1` detects collections correctly -- [ ] `extract-test-metadata.ps1` falls back to class mode when no collections -- [ ] `generate-test-matrix.ps1` creates valid JSON -- [ ] `generate-test-matrix.ps1` handles both collection and class modes - -### MSBuild Integration -- [ ] ExtractTestClassNames target executes -- [ ] `.tests.list` file is generated -- [ ] `.tests.metadata.json` file is generated -- [ ] Mode is correctly detected and stored in metadata -- [ ] GetTestProjects.proj identifies split projects - -### Generated Artifacts -- [ ] `.tests.list` format is correct -- [ ] `.tests.metadata.json` is valid JSON -- [ ] `split-tests-matrix.json` is valid JSON -- [ ] All matrix entries have required fields -- [ ] Filter arguments have correct syntax - -### xUnit Filters -- [ ] `--filter-collection` works -- [ ] `--filter-class` works -- [ ] `--filter-not-collection` works -- [ ] Filters run expected number of tests - -## Troubleshooting - -### Issue: "PowerShell script not found" - -**Error**: `Cannot find path 'eng/scripts/extract-test-metadata.ps1'` - -**Fix**: Ensure working directory is repository root: -```bash -cd /path/to/aspire -pwd # Should show aspire repo root -``` - -### Issue: "No tests found matching prefix" - -**Error**: `Error: No test classes found matching prefix` - -**Fix**: Verify `TestClassNamesPrefix` matches actual test namespace: -```bash -# Check test namespace -grep -r "^namespace " tests/YourProject.Tests/*.cs | head -1 -# Should match TestClassNamesPrefix -``` - -### Issue: "Mode is empty in metadata" - -**Error**: Mode field is empty or missing - -**Fix**: Check PowerShell script output - may have parsing errors. -Look in binlog for script console output. - -### Issue: "Matrix JSON is invalid" - -**Error**: GitHub Actions can't parse matrix - -**Fix**: Validate JSON locally: -```bash -jq empty artifacts/test-matrices/split-tests-matrix.json -# Exit code 0 = valid, non-zero = invalid -``` - -## Next Steps - -Once local testing passes: -1. Create PR with changes -2. Push to branch -3. Monitor GitHub Actions workflow -4. Verify matrices are generated correctly -5. Verify tests run in split jobs -6. Compare CI times before/after \ No newline at end of file diff --git a/docs/test-splitting/STEP_06_CI_INTEGRATION.md b/docs/test-splitting/STEP_06_CI_INTEGRATION.md deleted file mode 100644 index 3bbbc18466e..00000000000 --- a/docs/test-splitting/STEP_06_CI_INTEGRATION.md +++ /dev/null @@ -1,318 +0,0 @@ -# Step 6: CI Integration & Verification - -## Overview - -This guide explains how to verify the GitHub Actions integration and what to expect when your PR runs in CI. - -## GitHub Actions Workflow - -The existing `.github/workflows/tests.yml` workflow already supports the new matrix format from v1. No changes are needed because: - -1. The enumerate-tests action outputs `split_tests_matrix` -2. The workflow consumes it with `fromJson()` -3. The run-tests workflow accepts the matrix fields - -### Workflow Flow - -``` -setup_for_tests_lin (ubuntu-latest) - ↓ -enumerate-tests action - ↓ - ├─ Build split test projects - ├─ Call extract-test-metadata.ps1 - ├─ Call generate-test-matrix.ps1 - └─ Output: split_tests_matrix JSON - ↓ -split_tests_lin job - ↓ -Uses matrix: fromJson(needs.setup.outputs.split_tests_matrix) - ↓ -For each matrix entry: - - testShortName: ${{ matrix.shortname }} - - testProjectPath: ${{ matrix.testProjectPath }} - - extraTestArgs: ${{ matrix.extraTestArgs }} - - requiresNugets: ${{ matrix.requiresNugets }} - - etc. -``` - -### Key Matrix Fields Used by Workflow - -The workflow expects these fields (all present in v3 output): - -```yaml -matrix: - shortname: "Collection_DatabaseTests" # Used for job name - projectName: "Aspire.Hosting.Tests" # Used in extraTestArgs - testProjectPath: "tests/..." # Which project to test - extraTestArgs: "--filter-collection ..." # xUnit filter - requiresNugets: true/false # Download packages? - requiresTestSdk: true/false # Need test SDK? - testSessionTimeout: "20m" # Timeout - testHangTimeout: "10m" # Hang timeout - enablePlaywrightInstall: true/false # Install browsers? -``` - -All of these are generated by our scripts, so the workflow "just works". - -## What to Expect in CI - -### Setup Jobs (Per OS) - -**setup_for_tests_lin**, **setup_for_tests_macos**, **setup_for_tests_win** - -Each OS runs independently and generates its own matrix: - -``` -✓ Checkout code -✓ Set up .NET -✓ Generate test project lists - → Runs GetTestProjects.proj -✓ Build split test projects - → For each project in .split-projects - → Runs ExtractTestClassNames target - → Calls extract-test-metadata.ps1 -✓ Load split tests matrix - → Calls generate-test-matrix.ps1 - → Outputs JSON to GITHUB_OUTPUT -✓ Upload artifacts (binlogs, lists, matrices) -``` - -**Expected Duration**: 5-10 minutes per OS - -### Split Test Jobs - -**split_tests_lin**, **split_tests_macos**, **split_tests_win** - -If your project has splitting enabled, you'll see new jobs appear: - -**Collection Mode Example**: -``` -split_tests_lin / Aspire.Hosting.Tests_Collection_DatabaseTests (ubuntu-latest) -split_tests_lin / Aspire.Hosting.Tests_Collection_ContainerTests (ubuntu-latest) -split_tests_lin / Aspire.Hosting.Tests_Uncollected (ubuntu-latest) -``` - -**Class Mode Example**: -``` -split_tests_lin / Aspire.Templates.Tests_BuildAndRunTemplateTests (ubuntu-latest) -split_tests_lin / Aspire.Templates.Tests_EmptyTemplateRunTests (ubuntu-latest) -split_tests_lin / Aspire.Templates.Tests_StarterTemplateRunTests (ubuntu-latest) -... -``` - -Each job: -1. Downloads built packages (if `requiresNugets: true`) -2. Installs test SDK (if `requiresTestSdk: true`) -3. Runs: `dotnet test ... -- ` -4. Uploads test results - -**Expected Duration**: Varies by test, but should be significantly less than running all tests together - -## Monitoring Your PR - -### 1. Check Setup Jobs - -Navigate to your PR → Actions → Click on workflow run → Expand setup jobs - -**What to Look For**: -- [ ] "Build split test projects" step succeeds -- [ ] "Load split tests matrix" step outputs JSON -- [ ] Check artifacts → `logs-enumerate-tests-{OS}` contains: - - [ ] `.tests.list` files - - [ ] `.tests.metadata.json` files - - [ ] `split-tests-matrix.json` - -**Download and Inspect**: -```bash -# Download artifacts from GitHub UI -unzip logs-enumerate-tests-Linux.zip - -# Check generated files -cat artifacts/helix/*.tests.list -cat artifacts/helix/*.tests.metadata.json -cat artifacts/test-matrices/split-tests-matrix.json | jq . -``` - -### 2. Check Split Test Jobs - -Look for new jobs in the workflow run: - -**Collection Mode**: -- Job names like: `Split Tests Linux / {ProjectName}_Collection_{CollectionName}` -- Fewer jobs than test classes (grouped) - -**Class Mode**: -- Job names like: `Split Tests Linux / {ProjectName}_{ClassName}` -- One job per test class - -**What to Verify**: -- [ ] Jobs appear for each matrix entry -- [ ] Jobs run in parallel -- [ ] Each job uses correct filter argument -- [ ] Test results are uploaded -- [ ] All tests pass (or expected failures only) - -### 3. Compare CI Times - -**Before**: -``` -Aspire.Hosting.Tests (Linux): 1 job, 60 minutes -``` - -**After** (with collections): -``` -Collection_DatabaseTests: 25 minutes -Collection_ContainerTests: 20 minutes -Uncollected: 10 minutes -Total: ~25 minutes (parallel) -``` - -## Verification Checklist - -### Per-OS Setup (Run 3 times: Linux, macOS, Windows) - -- [ ] `setup_for_tests_{os}` job succeeds -- [ ] Split test projects are built -- [ ] Matrix JSON is generated and output -- [ ] Artifacts are uploaded - -### Split Test Execution (Per OS) - -- [ ] `split_tests_{os}` jobs appear -- [ ] Number of jobs matches matrix entries -- [ ] Each job runs correct filter -- [ ] Tests execute and pass -- [ ] Test results (.trx files) are uploaded - -### Matrix Validation - -- [ ] Download `split-tests-matrix.json` from artifacts -- [ ] Validate JSON structure: - ```bash - jq '.include | length' split-tests-matrix.json # Should be > 0 - jq '.include[0] | keys' split-tests-matrix.json # Check fields present - ``` -- [ ] Verify filter arguments are correct: - ```bash - jq '.include[] | {shortname, extraTestArgs}' split-tests-matrix.json - ``` - -## Common CI Issues - -### Issue 1: "No split test projects found" - -**Symptom**: Setup job completes but no split_tests_* jobs run - -**Cause**: No projects have `SplitTestsOnCI=true` set - -**Fix**: Verify `.csproj` has the property set - -### Issue 2: "Matrix is empty" - -**Symptom**: split_tests_* jobs are skipped - -**Cause**: Matrix generation failed or produced empty result - -**Fix**: -1. Download artifacts -2. Check if `.tests.list` files exist -3. Check if `split-tests-matrix.json` exists and has entries -4. Review binlogs for errors - -### Issue 3: "No tests executed" - -**Symptom**: Test job completes but .trx shows 0 tests - -**Cause**: Filter argument didn't match any tests - -**Fix**: -1. Check `extraTestArgs` in matrix JSON -2. Verify collection/class names match actual test code -3. Check `TestClassNamesPrefix` matches namespace - -### Issue 4: "Build failed for split project" - -**Symptom**: Setup job fails during "Build split test projects" - -**Cause**: Test project has build errors or missing dependencies - -**Fix**: -1. Check binlog: `Build_{ProjectName}.binlog` -2. Fix build errors -3. Test locally first with `dotnet build` - -## Rolling Back - -If issues arise in CI, you can disable splitting temporarily: - -### Option 1: Disable for One Project - -```xml - - -``` - -Push change → Project runs as single job again - -### Option 2: Disable Globally - -In `.github/workflows/tests.yml`, comment out split_tests_* jobs: - -```yaml - # split_tests_lin: - # uses: ./.github/workflows/run-tests.yml - # ... -``` - -This stops all split test execution (back to pre-PR behavior) - -## Success Metrics - -After your PR merges, track these metrics: - -### CI Time Reduction - -**Before**: Note longest test job duration -**After**: Note longest split test job duration -**Target**: 50%+ reduction - -Example: -``` -Before: Hosting.Tests = 60m -After: Collection_DatabaseTests = 25m (longest) -Improvement: 58% faster -``` - -### Job Count - -**Collection Mode**: Expect N+1 jobs (N collections + uncollected) -**Class Mode**: Expect N jobs (one per class) - -### Flakiness - -Monitor for: -- Tests failing intermittently in split jobs -- Tests passing in split jobs but failing when run together -- Resource contention issues (less likely with fewer tests per job) - -## Next Steps After CI Success - -1. **Monitor for 1-2 weeks** - - Watch for any new failures - - Check if CI times remain improved - - Look for resource issues - -2. **Enable for More Projects** - - Apply to other long-running test projects - - Add collections to optimize further - -3. **Document Learnings** - - Update best practices based on real usage - - Share collection grouping strategies - - Document any edge cases discovered - -4. **Optimize Further** - - Adjust collection groupings based on actual times - - Fine-tune timeouts - - Consider enabling for more projects \ No newline at end of file diff --git a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets index b84640de143..caa87ccad9f 100644 --- a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets +++ b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets @@ -15,24 +15,24 @@ - Builds the project and generates test metadata if needed - Writes .testenumeration.json file containing: * Project name, path, short name - * BuildOs (linux, windows, darwin) - * runOnGithubActions flag + * supportedOSes array * splitTests flag * hasTestMetadata flag * Paths to test list and metadata files - Phase 2 (AfterSolutionBuild.targets + process-test-enumeration.ps1): + Phase 2 (AfterSolutionBuild.targets + build-test-matrix.ps1): 1. After all projects are built: - Finds all .testenumeration.json files - - Filters by BuildOs and runOnGithubActions - - Separates regular vs split test projects - - Calls generate-test-matrix-for-split-tests.ps1 - - Generates combined test matrix JSON + - Filters by OS and processes regular vs split test projects + - For split tests: reads .tests.list and creates multiple matrix entries + - For regular tests: creates one matrix entry per project + - Generates combined test matrix JSON in a single pass Phase 3 (GitHub Actions): 1. Reads combined test matrix JSON - 2. Creates parallel jobs for each matrix entry - 3. Runs tests with appropriate filters/partitions + 2. Expands matrix entries by supported OSes + 3. Creates parallel jobs for each matrix entry + 4. Runs tests with appropriate filters/partitions Output per test project: - artifacts/tmp/Debug/ProjectName.testenumeration.json diff --git a/eng/scripts/generate-test-matrix.ps1 b/eng/scripts/generate-test-matrix.ps1 deleted file mode 100644 index 3fef7295cf4..00000000000 --- a/eng/scripts/generate-test-matrix.ps1 +++ /dev/null @@ -1,325 +0,0 @@ -<# -.SYNOPSIS - Generate split-tests matrix JSON supporting collection-based and class-based modes. - -.DESCRIPTION - Reads *.tests.list files: - collection mode format: - collection:Name - ... - uncollected:* (catch-all) - class mode format: - class:Full.Namespace.ClassName - - Builds matrix entries with fields consumed by CI: - type (collection | uncollected | class) - projectName - shortname - name - fullClassName (class mode only) - testProjectPath - extraTestArgs - requiresNugets - requiresTestSdk - enablePlaywrightInstall - testSessionTimeout - testHangTimeout - - Defaults (if metadata absent): - testSessionTimeout=20m - testHangTimeout=10m - uncollectedTestsSessionTimeout=15m - uncollectedTestsHangTimeout=10m - -.NOTES - PowerShell 7+, cross-platform. -#> - -[CmdletBinding()] -param( - [Parameter(Mandatory=$true)] - [string]$TestListsDirectory, - [Parameter(Mandatory=$true)] - [string]$OutputDirectory, - [Parameter(Mandatory=$false)] - [string]$RegularTestProjectsFile = '' -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version Latest - -# Define default values - only include properties in output when they differ from these -$script:Defaults = @{ - extraTestArgs = '' - requiresNugets = $false - requiresTestSdk = $false - enablePlaywrightInstall = $false - testSessionTimeout = '20m' - testHangTimeout = '10m' - supportedOSes = @('windows', 'linux', 'macos') -} - -function Read-Metadata($file, $projectName) { - $defaults = @{ - projectName = $projectName - testClassNamesPrefix = $projectName - testProjectPath = "tests/$projectName/$projectName.csproj" - extraTestArgs = '' - requiresNugets = 'false' - requiresTestSdk = 'false' - enablePlaywrightInstall = 'false' - testSessionTimeout = '20m' - testHangTimeout = '10m' - uncollectedTestsSessionTimeout = '15m' - uncollectedTestsHangTimeout = '10m' - supportedOSes = @('windows', 'linux', 'macos') - } - if (-not (Test-Path $file)) { return $defaults } - try { - $json = Get-Content -Raw -Path $file | ConvertFrom-Json - foreach ($k in $json.PSObject.Properties.Name) { - $defaults[$k] = $json.$k - } - } catch { - throw "Failed parsing metadata for ${projectName}: $_" - } - return $defaults -} - -function Add-OptionalProperty($entry, $key, $value, $default) { - # Only add property if it differs from the default - if ($null -ne $default) { - if ($value -is [Array] -and $default -is [Array]) { - # Compare arrays - if (($value.Count -ne $default.Count) -or (Compare-Object $value $default)) { - $entry[$key] = $value - } - } elseif ($value -ne $default) { - $entry[$key] = $value - } - } else { - # No default, always include - $entry[$key] = $value - } -} - -function New-EntryCollection($c,$meta) { - $projectShortName = $meta.projectName -replace '^Aspire\.' -replace '\.Tests$' - $extraTestArgsValue = "--filter-trait `"Partition=$c`"" - - $entry = [ordered]@{ - type = 'collection' - projectName = $meta.projectName - name = $c - shortname = "${projectShortName}_$c" - testProjectPath = $meta.testProjectPath - } - - # Add optional properties only if they differ from defaults - Add-OptionalProperty $entry 'extraTestArgs' $extraTestArgsValue $script:Defaults.extraTestArgs - Add-OptionalProperty $entry 'requiresNugets' ($meta.requiresNugets -eq 'true') $script:Defaults.requiresNugets - Add-OptionalProperty $entry 'requiresTestSdk' ($meta.requiresTestSdk -eq 'true') $script:Defaults.requiresTestSdk - Add-OptionalProperty $entry 'enablePlaywrightInstall' ($meta.enablePlaywrightInstall -eq 'true') $script:Defaults.enablePlaywrightInstall - Add-OptionalProperty $entry 'testSessionTimeout' $meta.testSessionTimeout $script:Defaults.testSessionTimeout - Add-OptionalProperty $entry 'testHangTimeout' $meta.testHangTimeout $script:Defaults.testHangTimeout - Add-OptionalProperty $entry 'supportedOSes' $meta.supportedOSes $script:Defaults.supportedOSes - - return $entry -} - -function New-EntryUncollected($collections,$meta) { - $filters = @() - foreach ($c in $collections) { - $filters += "--filter-not-trait `"Partition=$c`"" - } - $extraTestArgsValue = ($filters -join ' ') - - $entry = [ordered]@{ - type = 'uncollected' - projectName = $meta.projectName - name = 'UncollectedTests' - shortname = 'Uncollected' - testProjectPath = $meta.testProjectPath - } - - # Add optional properties only if they differ from defaults - # Note: uncollected tests may have different timeout defaults - $uncollectedSessionTimeout = $meta.uncollectedTestsSessionTimeout ?? $meta.testSessionTimeout - $uncollectedHangTimeout = $meta.uncollectedTestsHangTimeout ?? $meta.testHangTimeout - - Add-OptionalProperty $entry 'extraTestArgs' $extraTestArgsValue $script:Defaults.extraTestArgs - Add-OptionalProperty $entry 'requiresNugets' ($meta.requiresNugets -eq 'true') $script:Defaults.requiresNugets - Add-OptionalProperty $entry 'requiresTestSdk' ($meta.requiresTestSdk -eq 'true') $script:Defaults.requiresTestSdk - Add-OptionalProperty $entry 'enablePlaywrightInstall' ($meta.enablePlaywrightInstall -eq 'true') $script:Defaults.enablePlaywrightInstall - Add-OptionalProperty $entry 'testSessionTimeout' $uncollectedSessionTimeout $script:Defaults.testSessionTimeout - Add-OptionalProperty $entry 'testHangTimeout' $uncollectedHangTimeout $script:Defaults.testHangTimeout - Add-OptionalProperty $entry 'supportedOSes' $meta.supportedOSes $script:Defaults.supportedOSes - - return $entry -} - -function New-EntryClass($full,$meta) { - $prefix = $meta.testClassNamesPrefix - $short = $full - if ($prefix -and $full.StartsWith("$prefix.")) { - $short = $full.Substring($prefix.Length + 1) - } - $extraTestArgsValue = "--filter-class `"$full`"" - - $entry = [ordered]@{ - type = 'class' - projectName = $meta.projectName - name = $short - shortname = $short - fullClassName = $full - testProjectPath = $meta.testProjectPath - } - - # Add optional properties only if they differ from defaults - Add-OptionalProperty $entry 'extraTestArgs' $extraTestArgsValue $script:Defaults.extraTestArgs - Add-OptionalProperty $entry 'requiresNugets' ($meta.requiresNugets -eq 'true') $script:Defaults.requiresNugets - Add-OptionalProperty $entry 'requiresTestSdk' ($meta.requiresTestSdk -eq 'true') $script:Defaults.requiresTestSdk - Add-OptionalProperty $entry 'enablePlaywrightInstall' ($meta.enablePlaywrightInstall -eq 'true') $script:Defaults.enablePlaywrightInstall - Add-OptionalProperty $entry 'testSessionTimeout' $meta.testSessionTimeout $script:Defaults.testSessionTimeout - Add-OptionalProperty $entry 'testHangTimeout' $meta.testHangTimeout $script:Defaults.testHangTimeout - Add-OptionalProperty $entry 'supportedOSes' $meta.supportedOSes $script:Defaults.supportedOSes - - return $entry -} - -function New-EntryRegular($shortName) { - $entry = [ordered]@{ - type = 'regular' - projectName = "Aspire.$shortName.Tests" - name = $shortName - shortname = $shortName - testProjectPath = "tests/Aspire.$shortName.Tests/Aspire.$shortName.Tests.csproj" - } - - # All defaults match, so no need to add any optional properties - # (extraTestArgs is empty, which matches the default) - - return $entry -}if (-not (Test-Path $TestListsDirectory)) { - throw "Test lists directory not found: $TestListsDirectory" -} - -$listFiles = @(Get-ChildItem -Path $TestListsDirectory -Filter '*.tests.list' -Recurse -ErrorAction SilentlyContinue) -if ($listFiles.Count -eq 0) { - $empty = @{ include = @() } - New-Item -ItemType Directory -Force -Path $OutputDirectory | Out-Null - $empty | ConvertTo-Json -Depth 5 -Compress | Set-Content -Path (Join-Path $OutputDirectory 'combined-tests-matrix.json') -Encoding UTF8 - Write-Host "Empty matrix written (no .tests.list files)." - exit 0 -} - -$entries = [System.Collections.Generic.List[object]]::new() - -foreach ($lf in $listFiles) { - $fileName = $lf.Name -replace '\.tests\.list$','' - $projectName = $fileName - $lines = @(Get-Content $lf.FullName | Where-Object { $_ -and -not [string]::IsNullOrWhiteSpace($_) }) - $metadataPath = ($lf.FullName -replace '\.tests\.list$', '.tests.metadata.json') - $meta = Read-Metadata $metadataPath $projectName - if ($lines.Count -eq 0) { continue } - - if ($lines[0].StartsWith('collection:') -or $lines[0].StartsWith('uncollected:')) { - # collection mode - $collections = @() - $hasUncollected = $false - foreach ($l in $lines) { - if ($l -match '^collection:(.+)$') { $collections += $Matches[1].Trim() } - elseif ($l -match '^uncollected:') { $hasUncollected = $true } - } - foreach ($c in ($collections | Sort-Object)) { - $entries.Add( (New-EntryCollection $c $meta) ) | Out-Null - } - if ($hasUncollected) { - $entries.Add( (New-EntryUncollected $collections $meta) ) | Out-Null - } - } elseif ($lines[0].StartsWith('class:')) { - # class mode - foreach ($l in $lines) { - if ($l -match '^class:(.+)$') { - $entries.Add( (New-EntryClass $Matches[1].Trim() $meta) ) | Out-Null - } - } - } -} - -# Add regular (non-split) test projects if provided -if ($RegularTestProjectsFile -and (Test-Path $RegularTestProjectsFile)) { - # Check if JSON file exists with full metadata - $jsonFile = "$RegularTestProjectsFile.json" - if (Test-Path $jsonFile) { - $regularProjectsData = Get-Content -Raw $jsonFile | ConvertFrom-Json - if ($regularProjectsData -isnot [Array]) { - $regularProjectsData = @($regularProjectsData) - } - Write-Host "Adding $($regularProjectsData.Count) regular test project(s) from JSON" - foreach ($proj in $regularProjectsData) { - # Try to read metadata file for this project if it exists - $metadataFile = $null - if ($proj.metadataFile) { - # metadataFile path is relative to repo root, so make it absolute - $metadataFile = Join-Path $TestListsDirectory ".." ($proj.metadataFile -replace '^artifacts/', '') - } - - $meta = $null - if ($metadataFile -and (Test-Path $metadataFile)) { - $meta = Read-Metadata $metadataFile $proj.project - #Write-Host " Loaded metadata for $($proj.project) from $metadataFile (requiresNugets=$($meta.requiresNugets))" - } else { - # Use defaults if no metadata file exists - # Note: supportedOSes comes from the project JSON, not defaults - $projectSupportedOSes = if ($proj.PSObject.Properties['supportedOSes']) { $proj.supportedOSes } else { @('windows', 'linux', 'macos') } - $meta = @{ - projectName = $proj.project - testProjectPath = $proj.fullPath - extraTestArgs = '' - requiresNugets = 'false' - requiresTestSdk = 'false' - enablePlaywrightInstall = 'false' - testSessionTimeout = '20m' - testHangTimeout = '10m' - supportedOSes = $projectSupportedOSes - } - Write-Host " Using default metadata for $($proj.project) (no metadata file found at $metadataFile)" - } - - $entry = [ordered]@{ - type = 'regular' - projectName = $proj.project - name = $proj.shortName - shortname = $proj.shortName - testProjectPath = $proj.fullPath - } - - # Add optional properties only if they differ from defaults - # Note: supportedOSes from the project JSON takes precedence - $finalSupportedOSes = if ($proj.PSObject.Properties['supportedOSes']) { $proj.supportedOSes } else { $meta.supportedOSes } - - Add-OptionalProperty $entry 'extraTestArgs' $meta.extraTestArgs $script:Defaults.extraTestArgs - Add-OptionalProperty $entry 'requiresNugets' ($meta.requiresNugets -eq 'true') $script:Defaults.requiresNugets - Add-OptionalProperty $entry 'requiresTestSdk' ($meta.requiresTestSdk -eq 'true') $script:Defaults.requiresTestSdk - Add-OptionalProperty $entry 'enablePlaywrightInstall' ($meta.enablePlaywrightInstall -eq 'true') $script:Defaults.enablePlaywrightInstall - Add-OptionalProperty $entry 'testSessionTimeout' $meta.testSessionTimeout $script:Defaults.testSessionTimeout - Add-OptionalProperty $entry 'testHangTimeout' $meta.testHangTimeout $script:Defaults.testHangTimeout - Add-OptionalProperty $entry 'supportedOSes' $finalSupportedOSes $script:Defaults.supportedOSes - - $entries.Add($entry) | Out-Null - } - } else { - # Fallback to old behavior for backward compatibility - $regularProjects = @(Get-Content $RegularTestProjectsFile | Where-Object { $_ -and -not [string]::IsNullOrWhiteSpace($_) }) - Write-Host "Adding $($regularProjects.Count) regular test project(s) (legacy mode)" - foreach ($shortName in $regularProjects) { - $entries.Add( (New-EntryRegular $shortName) ) | Out-Null - } - } -} - -$matrix = @{ include = $entries } -New-Item -ItemType Directory -Force -Path $OutputDirectory | Out-Null -$matrix | ConvertTo-Json -Depth 10 -Compress | Set-Content -Path (Join-Path $OutputDirectory 'combined-tests-matrix.json') -Encoding UTF8 -Write-Host "Matrix entries: $($entries.Count)" diff --git a/eng/scripts/process-test-enumeration.ps1 b/eng/scripts/process-test-enumeration.ps1 deleted file mode 100644 index 81a50e4e0f0..00000000000 --- a/eng/scripts/process-test-enumeration.ps1 +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env pwsh - -param( - [Parameter(Mandatory=$false)] - [string]$TestsListOutputPath, - - [Parameter(Mandatory=$false)] - [string]$TestMatrixOutputPath, - - [Parameter(Mandatory=$true)] - [string]$ArtifactsTmpDir, - - [Parameter(Mandatory=$true)] - [string]$RepoRoot -) - -Write-Host "Processing test enumeration files" -Write-Host "TestsListOutputPath: $TestsListOutputPath" -Write-Host "TestMatrixOutputPath: $TestMatrixOutputPath" -Write-Host "ArtifactsTmpDir: $ArtifactsTmpDir" - -# Find all test enumeration files -$enumerationFiles = Get-ChildItem -Path $ArtifactsTmpDir -Filter '*.testenumeration.json' -ErrorAction SilentlyContinue - -if (-not $enumerationFiles) { - Write-Error "No test enumeration files found in $ArtifactsTmpDir" - # Create empty output files - "" | Set-Content $TestsListOutputPath - if ($TestMatrixOutputPath) { - New-Item -Path $TestMatrixOutputPath -ItemType Directory -Force | Out-Null - } - exit 1 -} - -# Validate TestMatrixOutputPath if provided -# fail if empty -if ($TestMatrixOutputPath -and [string]::IsNullOrWhiteSpace($TestMatrixOutputPath)) { - Write-Error "TestMatrixOutputPath cannot be empty if provided" - exit 1 -} - -# TestMatrixOutputPath must be a JSON file path -if ($TestMatrixOutputPath -notmatch '\.json$') { - Write-Error "TestMatrixOutputPath must be a JSON file path: $TestMatrixOutputPath" - exit 1 -} - -# Check parent directory exists -$parentDir = Split-Path $TestMatrixOutputPath -Parent -if (-not (Test-Path $parentDir)) { - Write-Error "Parent directory for TestMatrixOutputPath does not exist: $parentDir" - exit 1 -} - -Write-Host "Found $($enumerationFiles.Count) test enumeration files" - -# Process enumeration files -$regularTestProjects = @() -$splitTestProjects = @() - -foreach ($file in $enumerationFiles) { - try { - $content = Get-Content -Raw $file.FullName | ConvertFrom-Json - - # Include all test projects that support at least one OS - if ($content.supportedOSes -and $content.supportedOSes.Count -gt 0) { - if ($content.splitTests -eq 'true') { - $splitTestProjects += $content.shortName - } else { - # Store full enumeration data for regular tests - $regularTestProjects += $content - } - $osesStr = $content.supportedOSes -join ', ' - Write-Host " Included: $($content.shortName) (OSes: $osesStr, Split: $($content.splitTests))" - } else { - Write-Host " Excluded: $($content.shortName) (No supported OSes)" - } - } - catch { - Write-Warning "Failed to process $($file.FullName): $_" - } -} - -Write-Host "Regular test projects: $($regularTestProjects.Count)" -Write-Host "Split test projects: $($splitTestProjects.Count)" - -# Create output directory if needed -$outputDir = Split-Path $TestsListOutputPath -Parent -if (-not (Test-Path $outputDir)) { - New-Item -Path $outputDir -ItemType Directory -Force | Out-Null -} - -# Write regular test projects list as JSON for matrix generation -if ($regularTestProjects.Count -gt 0) { - $regularTestProjects | ConvertTo-Json -Depth 10 | Set-Content "$TestsListOutputPath.json" -} -# Also write just the short names for backward compatibility -$regularTestProjects | ForEach-Object { $_.shortName } | Set-Content $TestsListOutputPath - -# Write split test projects list if any exist -if ($splitTestProjects.Count -gt 0) { - $splitTestProjects | Select-Object -Unique | Set-Content "$TestsListOutputPath.split-projects" - Write-Host "Split projects written to: $TestsListOutputPath.split-projects" -} -else { - Write-Host "No split test projects found, skipping split-projects file creation" -} - - -Write-Host "Generating test matrices..." - -# Create directory for intermediate files -$tempMatrixDir = Join-Path (Split-Path $TestMatrixOutputPath -Parent) 'temp-matrix' -New-Item -Path $tempMatrixDir -ItemType Directory -Force | Out-Null - -# Call existing matrix generation script if split tests exist -$matrixScriptPath = Join-Path $RepoRoot 'eng/scripts/generate-test-matrix.ps1' -$testListsDir = Join-Path (Split-Path $TestsListOutputPath -Parent) 'helix' -Write-Host "Calling matrix generation script..." -& $matrixScriptPath -TestListsDirectory $testListsDir -OutputDirectory $tempMatrixDir -RegularTestProjectsFile $TestsListOutputPath - -# Copy the generated matrix file to the expected location -$generatedMatrixFile = Join-Path $tempMatrixDir 'combined-tests-matrix.json' -if (Test-Path $generatedMatrixFile) { - Copy-Item $generatedMatrixFile $TestMatrixOutputPath - Write-Host "Matrix file copied to: $TestMatrixOutputPath" -} else { - Write-Error "Expected matrix file not found at: $generatedMatrixFile" - exit 1 -} - -# Clean up temporary directory -Remove-Item $tempMatrixDir -Recurse -Force -ErrorAction SilentlyContinue - -Write-Host "Test enumeration processing completed" -Write-Host "Regular projects written to: $TestsListOutputPath" -#if ($splitTestProjects.Count -gt 0) { - #Write-Host "Split projects written to: $TestsListOutputPath.split-projects" -#} -if ($TestMatrixOutputPath) { - Write-Host "Test matrices written to: $TestMatrixOutputPath" -} From fa289678908c948488935bf7f2c57b738e91bcf1 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 19:38:17 -0400 Subject: [PATCH 38/48] cleanup --- eng/scripts/build-test-matrix.ps1 | 4 ++-- ...tract-test-metadata.ps1 => split-test-projects-for-ci.ps1} | 0 tests/Directory.Build.targets | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename eng/scripts/{extract-test-metadata.ps1 => split-test-projects-for-ci.ps1} (100%) diff --git a/eng/scripts/build-test-matrix.ps1 b/eng/scripts/build-test-matrix.ps1 index 40a914759cb..ef00671e454 100644 --- a/eng/scripts/build-test-matrix.ps1 +++ b/eng/scripts/build-test-matrix.ps1 @@ -117,7 +117,7 @@ function New-CollectionTestEntry { $entry = [ordered]@{ type = 'collection' project = $Metadata.projectName - shortname = "$($Metadata.projectName)-$suffix" + shortname = if ($IsUncollected) { $Metadata.projectName } else { "$($Metadata.projectName)-$suffix" } testProjectPath = $Metadata.testProjectPath workitemprefix = "$($Metadata.projectName)_$suffix" collection = $CollectionName @@ -147,7 +147,7 @@ function New-CollectionTestEntry { # Add test filter for collection-based splitting if ($IsUncollected) { - $entry['extraTestArgs'] = '--filter-not-trait "Partition"' + $entry['extraTestArgs'] = '--filter-not-trait "Partition=*"' } else { $entry['extraTestArgs'] = "--filter-trait `"Partition=$CollectionName`"" } diff --git a/eng/scripts/extract-test-metadata.ps1 b/eng/scripts/split-test-projects-for-ci.ps1 similarity index 100% rename from eng/scripts/extract-test-metadata.ps1 rename to eng/scripts/split-test-projects-for-ci.ps1 diff --git a/tests/Directory.Build.targets b/tests/Directory.Build.targets index 2b77b28907c..2c3321aa7a4 100644 --- a/tests/Directory.Build.targets +++ b/tests/Directory.Build.targets @@ -49,7 +49,7 @@ - <_DiscoveryScriptPath>$(RepoRoot)eng\scripts\extract-test-metadata.ps1 + <_DiscoveryScriptPath>$(RepoRoot)eng\scripts\split-test-projects-for-ci.ps1 <_TestListFile>$(TestArchiveTestsDir)$(MSBuildProjectName).tests.list From 7b26f9d3bbe0ba55f12eb28df9effa48a2f0fd5c Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 20:00:55 -0400 Subject: [PATCH 39/48] cleanup --- eng/Testing.props | 6 ++++ eng/Testing.targets | 29 ++++++++++++++----- eng/scripts/build-test-matrix.ps1 | 8 +++-- .../Aspire.Hosting.Tests.csproj | 4 +-- .../Aspire.Templates.Tests.csproj | 4 +-- tests/Directory.Build.targets | 4 +-- 6 files changed, 38 insertions(+), 17 deletions(-) diff --git a/eng/Testing.props b/eng/Testing.props index d24ccae9410..5fad18f14ca 100644 --- a/eng/Testing.props +++ b/eng/Testing.props @@ -2,6 +2,12 @@ true + + 20m + 10m + 15m + 10m + true true diff --git a/eng/Testing.targets b/eng/Testing.targets index 11b1f92598b..6373508cf97 100644 --- a/eng/Testing.targets +++ b/eng/Testing.targets @@ -210,15 +210,24 @@ <_SupportedOSesJson Condition="'$(_SupportedOSesJson)' != ''">$(_SupportedOSesJson.TrimEnd(',')) - - <_TestSessionTimeout Condition="'$(SplitTestSessionTimeout)' != ''">$(SplitTestSessionTimeout) - <_TestSessionTimeout Condition="'$(SplitTestSessionTimeout)' == ''">20m - <_TestHangTimeout Condition="'$(SplitTestHangTimeout)' != ''">$(SplitTestHangTimeout) - <_TestHangTimeout Condition="'$(SplitTestHangTimeout)' == ''">10m + + <_TestSessionTimeout Condition="'$(TestSessionTimeout)' != ''">$(TestSessionTimeout) + <_TestSessionTimeout Condition="'$(TestSessionTimeout)' == ''">20m + <_TestHangTimeout Condition="'$(TestHangTimeout)' != ''">$(TestHangTimeout) + <_TestHangTimeout Condition="'$(TestHangTimeout)' == ''">10m + <_UncollectedTestsSessionTimeout Condition="'$(UncollectedTestsSessionTimeout)' != ''">$(UncollectedTestsSessionTimeout) + <_UncollectedTestsSessionTimeout Condition="'$(UncollectedTestsSessionTimeout)' == ''">15m + <_UncollectedTestsHangTimeout Condition="'$(UncollectedTestsHangTimeout)' != ''">$(UncollectedTestsHangTimeout) + <_UncollectedTestsHangTimeout Condition="'$(UncollectedTestsHangTimeout)' == ''">10m + + + + <_ShortName>$([System.IO.Path]::GetFileNameWithoutExtension('$(MSBuildProjectName)').Replace('Aspire.', '').Replace('.Tests', '')) <_MetadataJson>{ "projectName": "$(MSBuildProjectName)", + "shortName": "$(_ShortName)", "testClassNamesPrefix": "$(MSBuildProjectName)", "testProjectPath": "$(_RelativeProjectPath)", "requiresNugets": "$(_RequiresNugets.ToLowerInvariant())", @@ -234,9 +243,13 @@ - + diff --git a/eng/scripts/build-test-matrix.ps1 b/eng/scripts/build-test-matrix.ps1 index ef00671e454..f5ff0d5156d 100644 --- a/eng/scripts/build-test-matrix.ps1 +++ b/eng/scripts/build-test-matrix.ps1 @@ -23,7 +23,7 @@ Path to write the combined test matrix JSON file. .PARAMETER TestsListOutputFile - Optional path to write backward-compatible test list file (regular tests only). + Optional path to write backward-compatible test list file (regular tests only) used on AzDO .PARAMETER CurrentOS Current operating system (linux, windows, macos). Filters tests by supported OSes. @@ -113,11 +113,12 @@ function New-CollectionTestEntry { ) $suffix = if ($IsUncollected) { 'uncollected' } else { $CollectionName } + $baseShortName = if ($Metadata.shortName) { $Metadata.shortName } else { $Metadata.projectName } $entry = [ordered]@{ type = 'collection' project = $Metadata.projectName - shortname = if ($IsUncollected) { $Metadata.projectName } else { "$($Metadata.projectName)-$suffix" } + shortname = if ($IsUncollected) { "$baseShortName-$suffix" } else { "$baseShortName-$suffix" } testProjectPath = $Metadata.testProjectPath workitemprefix = "$($Metadata.projectName)_$suffix" collection = $CollectionName @@ -171,11 +172,12 @@ function New-ClassTestEntry { # Extract short class name (last segment after last dot) $shortClassName = $ClassName.Split('.')[-1] + $baseShortName = if ($Metadata.shortName) { $Metadata.shortName } else { $Metadata.projectName } $entry = [ordered]@{ type = 'class' project = $Metadata.projectName - shortname = "$($Metadata.projectName)-$shortClassName" + shortname = "$baseShortName-$shortClassName" testProjectPath = $Metadata.testProjectPath workitemprefix = "$($Metadata.projectName)_$shortClassName" classname = $ClassName diff --git a/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj b/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj index b2d4d254b01..f5c00c87ed7 100644 --- a/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj +++ b/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj @@ -16,8 +16,8 @@ Aspire.Hosting.Tests - 30m - 15m + 30m + 15m 20m 10m diff --git a/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj b/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj index 39faf84fdbf..31aca3598d5 100644 --- a/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj +++ b/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj @@ -23,8 +23,8 @@ true - 20m - 12m + 20m + 15m - + From eb7c9b89fe9595c77145bb3f70d059c50f75964f Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 21:14:41 -0400 Subject: [PATCH 41/48] fix yml --- .github/actions/enumerate-tests/action.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/actions/enumerate-tests/action.yml b/.github/actions/enumerate-tests/action.yml index 14b97b16f90..764747b1b47 100644 --- a/.github/actions/enumerate-tests/action.yml +++ b/.github/actions/enumerate-tests/action.yml @@ -54,9 +54,9 @@ runs: # Define defaults to apply when properties are missing $defaults = @{ extraTestArgs = '' - requiresNugets = 'false' - requiresTestSdk = 'false' - enablePlaywrightInstall = 'false' + requiresNugets = $false + requiresTestSdk = $false + enablePlaywrightInstall = $false testSessionTimeout = '20m' testHangTimeout = '10m' supportedOSes = @('windows', 'linux', 'macos') @@ -100,19 +100,19 @@ runs: } $testCopy | Add-Member -NotePropertyName 'runs-on' -NotePropertyValue $osRunner -Force - # Normalize boolean string values to ensure consistency + # Normalize boolean values to actual booleans (not strings) if ($testCopy.PSObject.Properties.Name.Contains('requiresNugets')) { - $testCopy.requiresNugets = ($testCopy.requiresNugets -eq 'true' -or $testCopy.requiresNugets -eq $true).ToString().ToLower() + $testCopy.requiresNugets = ($testCopy.requiresNugets -eq 'true' -or $testCopy.requiresNugets -eq $true) } if ($testCopy.PSObject.Properties.Name.Contains('requiresTestSdk')) { - $testCopy.requiresTestSdk = ($testCopy.requiresTestSdk -eq 'true' -or $testCopy.requiresTestSdk -eq $true).ToString().ToLower() + $testCopy.requiresTestSdk = ($testCopy.requiresTestSdk -eq 'true' -or $testCopy.requiresTestSdk -eq $true) } if ($testCopy.PSObject.Properties.Name.Contains('enablePlaywrightInstall')) { - $testCopy.enablePlaywrightInstall = ($testCopy.enablePlaywrightInstall -eq 'true' -or $testCopy.enablePlaywrightInstall -eq $true).ToString().ToLower() + $testCopy.enablePlaywrightInstall = ($testCopy.enablePlaywrightInstall -eq 'true' -or $testCopy.enablePlaywrightInstall -eq $true) } # Add to appropriate list based on requiresNugets - if ($testCopy.requiresNugets -eq 'true') { + if ($testCopy.requiresNugets -eq $true) { $testsRequiringNugets += $testCopy } else { $testsNotRequiringNugets += $testCopy From 36d080858443390a37d79a0d7d2063ac8d5b5b5e Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 21:19:38 -0400 Subject: [PATCH 42/48] improvey --- .github/workflows/tests.yml | 2 +- eng/scripts/build-test-matrix.ps1 | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4e60f65a5e9..f6af3bd3825 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -82,7 +82,7 @@ jobs: - name: Setup Node.js environment uses: actions/setup-node@v2 with: - node-version: ${{ matrix.node-version }} + node-version: '20' - name: Install dependencies run: yarn install - name: Run tests diff --git a/eng/scripts/build-test-matrix.ps1 b/eng/scripts/build-test-matrix.ps1 index f5ff0d5156d..251137cc5ee 100644 --- a/eng/scripts/build-test-matrix.ps1 +++ b/eng/scripts/build-test-matrix.ps1 @@ -80,7 +80,8 @@ function New-RegularTestEntry { $entry = [ordered]@{ type = 'regular' - project = $Enumeration.project + projectName = $Enumeration.project + name = $Enumeration.shortName shortname = $Enumeration.shortName testProjectPath = $Enumeration.fullPath workitemprefix = $Enumeration.project @@ -93,6 +94,7 @@ function New-RegularTestEntry { if ($Metadata.requiresNugets -eq 'true') { $entry['requiresNugets'] = 'true' } if ($Metadata.requiresTestSdk -eq 'true') { $entry['requiresTestSdk'] = 'true' } if ($Metadata.enablePlaywrightInstall -eq 'true') { $entry['enablePlaywrightInstall'] = 'true' } + if ($Metadata.extraTestArgs) { $entry['extraTestArgs'] = $Metadata.extraTestArgs } } # Add supported OSes @@ -117,7 +119,8 @@ function New-CollectionTestEntry { $entry = [ordered]@{ type = 'collection' - project = $Metadata.projectName + projectName = $Metadata.projectName + name = if ($IsUncollected) { "$baseShortName-$suffix" } else { "$baseShortName-$suffix" } shortname = if ($IsUncollected) { "$baseShortName-$suffix" } else { "$baseShortName-$suffix" } testProjectPath = $Metadata.testProjectPath workitemprefix = "$($Metadata.projectName)_$suffix" @@ -176,7 +179,8 @@ function New-ClassTestEntry { $entry = [ordered]@{ type = 'class' - project = $Metadata.projectName + projectName = $Metadata.projectName + name = "$baseShortName-$shortClassName" shortname = "$baseShortName-$shortClassName" testProjectPath = $Metadata.testProjectPath workitemprefix = "$($Metadata.projectName)_$shortClassName" From 6a3efd4f08ac03439e5102d039e31ae1f3b14760 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 21:42:25 -0400 Subject: [PATCH 43/48] fix name --- eng/scripts/build-test-matrix.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/scripts/build-test-matrix.ps1 b/eng/scripts/build-test-matrix.ps1 index 251137cc5ee..15cbd6b8edb 100644 --- a/eng/scripts/build-test-matrix.ps1 +++ b/eng/scripts/build-test-matrix.ps1 @@ -120,8 +120,8 @@ function New-CollectionTestEntry { $entry = [ordered]@{ type = 'collection' projectName = $Metadata.projectName - name = if ($IsUncollected) { "$baseShortName-$suffix" } else { "$baseShortName-$suffix" } - shortname = if ($IsUncollected) { "$baseShortName-$suffix" } else { "$baseShortName-$suffix" } + name = if ($IsUncollected) { $baseShortName } else { "$baseShortName-$suffix" } + shortname = if ($IsUncollected) { $baseShortName } else { "$baseShortName-$suffix" } testProjectPath = $Metadata.testProjectPath workitemprefix = "$($Metadata.projectName)_$suffix" collection = $CollectionName From 813144af3919e39ab30ad6f6b7bfe2e00f93a4aa Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Oct 2025 21:52:13 -0400 Subject: [PATCH 44/48] fix ps1 --- .github/workflows/tests.yml | 2 +- eng/scripts/build-test-matrix.ps1 | 44 +++++++++++++++---------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f6af3bd3825..0938c6d8cb3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,7 +32,7 @@ jobs: tests_no_nugets: uses: ./.github/workflows/run-tests.yml - name: Tests (No Nugets) + name: Tests needs: setup_for_tests if: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets).include[0] != null }} strategy: diff --git a/eng/scripts/build-test-matrix.ps1 b/eng/scripts/build-test-matrix.ps1 index 15cbd6b8edb..431e4e31762 100644 --- a/eng/scripts/build-test-matrix.ps1 +++ b/eng/scripts/build-test-matrix.ps1 @@ -89,12 +89,12 @@ function New-RegularTestEntry { # Add metadata if available if ($Metadata) { - if ($Metadata.testSessionTimeout) { $entry['testSessionTimeout'] = $Metadata.testSessionTimeout } - if ($Metadata.testHangTimeout) { $entry['testHangTimeout'] = $Metadata.testHangTimeout } - if ($Metadata.requiresNugets -eq 'true') { $entry['requiresNugets'] = 'true' } - if ($Metadata.requiresTestSdk -eq 'true') { $entry['requiresTestSdk'] = 'true' } - if ($Metadata.enablePlaywrightInstall -eq 'true') { $entry['enablePlaywrightInstall'] = 'true' } - if ($Metadata.extraTestArgs) { $entry['extraTestArgs'] = $Metadata.extraTestArgs } + if ($Metadata.PSObject.Properties['testSessionTimeout']) { $entry['testSessionTimeout'] = $Metadata.testSessionTimeout } + if ($Metadata.PSObject.Properties['testHangTimeout']) { $entry['testHangTimeout'] = $Metadata.testHangTimeout } + if ($Metadata.PSObject.Properties['requiresNugets'] -and $Metadata.requiresNugets -eq 'true') { $entry['requiresNugets'] = 'true' } + if ($Metadata.PSObject.Properties['requiresTestSdk'] -and $Metadata.requiresTestSdk -eq 'true') { $entry['requiresTestSdk'] = 'true' } + if ($Metadata.PSObject.Properties['enablePlaywrightInstall'] -and $Metadata.enablePlaywrightInstall -eq 'true') { $entry['enablePlaywrightInstall'] = 'true' } + if ($Metadata.PSObject.Properties['extraTestArgs'] -and $Metadata.extraTestArgs) { $entry['extraTestArgs'] = $Metadata.extraTestArgs } } # Add supported OSes @@ -129,25 +129,25 @@ function New-CollectionTestEntry { # Use uncollected timeouts if available, otherwise use regular if ($IsUncollected) { - if ($Metadata.uncollectedTestsSessionTimeout) { + if ($Metadata.PSObject.Properties['uncollectedTestsSessionTimeout']) { $entry['testSessionTimeout'] = $Metadata.uncollectedTestsSessionTimeout - } elseif ($Metadata.testSessionTimeout) { + } elseif ($Metadata.PSObject.Properties['testSessionTimeout']) { $entry['testSessionTimeout'] = $Metadata.testSessionTimeout } - if ($Metadata.uncollectedTestsHangTimeout) { + if ($Metadata.PSObject.Properties['uncollectedTestsHangTimeout']) { $entry['testHangTimeout'] = $Metadata.uncollectedTestsHangTimeout - } elseif ($Metadata.testHangTimeout) { + } elseif ($Metadata.PSObject.Properties['testHangTimeout']) { $entry['testHangTimeout'] = $Metadata.testHangTimeout } } else { - if ($Metadata.testSessionTimeout) { $entry['testSessionTimeout'] = $Metadata.testSessionTimeout } - if ($Metadata.testHangTimeout) { $entry['testHangTimeout'] = $Metadata.testHangTimeout } + if ($Metadata.PSObject.Properties['testSessionTimeout']) { $entry['testSessionTimeout'] = $Metadata.testSessionTimeout } + if ($Metadata.PSObject.Properties['testHangTimeout']) { $entry['testHangTimeout'] = $Metadata.testHangTimeout } } - if ($Metadata.requiresNugets -eq 'true') { $entry['requiresNugets'] = 'true' } - if ($Metadata.requiresTestSdk -eq 'true') { $entry['requiresTestSdk'] = 'true' } - if ($Metadata.enablePlaywrightInstall -eq 'true') { $entry['enablePlaywrightInstall'] = 'true' } + if ($Metadata.PSObject.Properties['requiresNugets'] -and $Metadata.requiresNugets -eq 'true') { $entry['requiresNugets'] = 'true' } + if ($Metadata.PSObject.Properties['requiresTestSdk'] -and $Metadata.requiresTestSdk -eq 'true') { $entry['requiresTestSdk'] = 'true' } + if ($Metadata.PSObject.Properties['enablePlaywrightInstall'] -and $Metadata.enablePlaywrightInstall -eq 'true') { $entry['enablePlaywrightInstall'] = 'true' } # Add test filter for collection-based splitting if ($IsUncollected) { @@ -157,7 +157,7 @@ function New-CollectionTestEntry { } # Add supported OSes from metadata (should match enumeration) - if ($Metadata.supportedOSes) { + if ($Metadata.PSObject.Properties['supportedOSes']) { $entry['supportedOSes'] = @($Metadata.supportedOSes) } @@ -187,17 +187,17 @@ function New-ClassTestEntry { classname = $ClassName } - if ($Metadata.testSessionTimeout) { $entry['testSessionTimeout'] = $Metadata.testSessionTimeout } - if ($Metadata.testHangTimeout) { $entry['testHangTimeout'] = $Metadata.testHangTimeout } - if ($Metadata.requiresNugets -eq 'true') { $entry['requiresNugets'] = 'true' } - if ($Metadata.requiresTestSdk -eq 'true') { $entry['requiresTestSdk'] = 'true' } - if ($Metadata.enablePlaywrightInstall -eq 'true') { $entry['enablePlaywrightInstall'] = 'true' } + if ($Metadata.PSObject.Properties['testSessionTimeout']) { $entry['testSessionTimeout'] = $Metadata.testSessionTimeout } + if ($Metadata.PSObject.Properties['testHangTimeout']) { $entry['testHangTimeout'] = $Metadata.testHangTimeout } + if ($Metadata.PSObject.Properties['requiresNugets'] -and $Metadata.requiresNugets -eq 'true') { $entry['requiresNugets'] = 'true' } + if ($Metadata.PSObject.Properties['requiresTestSdk'] -and $Metadata.requiresTestSdk -eq 'true') { $entry['requiresTestSdk'] = 'true' } + if ($Metadata.PSObject.Properties['enablePlaywrightInstall'] -and $Metadata.enablePlaywrightInstall -eq 'true') { $entry['enablePlaywrightInstall'] = 'true' } # Add test filter for class-based splitting $entry['extraTestArgs'] = "--filter-class `"$ClassName`"" # Add supported OSes from metadata - if ($Metadata.supportedOSes) { + if ($Metadata.PSObject.Properties['supportedOSes']) { $entry['supportedOSes'] = @($Metadata.supportedOSes) } From 44af4bf75cb0d79bff6e234782327c6d3debd6d9 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sat, 18 Oct 2025 00:02:03 -0400 Subject: [PATCH 45/48] fixup --- .github/actions/enumerate-tests/action.yml | 4 ---- .github/workflows/tests.yml | 2 -- eng/scripts/build-test-matrix.ps1 | 3 --- tests/Directory.Build.targets | 3 +-- 4 files changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/actions/enumerate-tests/action.yml b/.github/actions/enumerate-tests/action.yml index 764747b1b47..d84fae85e61 100644 --- a/.github/actions/enumerate-tests/action.yml +++ b/.github/actions/enumerate-tests/action.yml @@ -56,7 +56,6 @@ runs: extraTestArgs = '' requiresNugets = $false requiresTestSdk = $false - enablePlaywrightInstall = $false testSessionTimeout = '20m' testHangTimeout = '10m' supportedOSes = @('windows', 'linux', 'macos') @@ -107,9 +106,6 @@ runs: if ($testCopy.PSObject.Properties.Name.Contains('requiresTestSdk')) { $testCopy.requiresTestSdk = ($testCopy.requiresTestSdk -eq 'true' -or $testCopy.requiresTestSdk -eq $true) } - if ($testCopy.PSObject.Properties.Name.Contains('enablePlaywrightInstall')) { - $testCopy.enablePlaywrightInstall = ($testCopy.enablePlaywrightInstall -eq 'true' -or $testCopy.enablePlaywrightInstall -eq $true) - } # Add to appropriate list based on requiresNugets if ($testCopy.requiresNugets -eq $true) { diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0938c6d8cb3..88cf2764ca1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,7 +48,6 @@ jobs: versionOverrideArg: ${{ inputs.versionOverrideArg }} requiresNugets: ${{ matrix.requiresNugets }} requiresTestSdk: ${{ matrix.requiresTestSdk }} - enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} tests_requires_nugets: name: Tests (Requires Nugets) @@ -68,7 +67,6 @@ jobs: versionOverrideArg: ${{ inputs.versionOverrideArg }} requiresNugets: ${{ matrix.requiresNugets }} requiresTestSdk: ${{ matrix.requiresTestSdk }} - enablePlaywrightInstall: ${{ matrix.enablePlaywrightInstall }} extension_tests_win: name: Run VS Code extension tests (Windows) diff --git a/eng/scripts/build-test-matrix.ps1 b/eng/scripts/build-test-matrix.ps1 index 431e4e31762..9cef64e3179 100644 --- a/eng/scripts/build-test-matrix.ps1 +++ b/eng/scripts/build-test-matrix.ps1 @@ -93,7 +93,6 @@ function New-RegularTestEntry { if ($Metadata.PSObject.Properties['testHangTimeout']) { $entry['testHangTimeout'] = $Metadata.testHangTimeout } if ($Metadata.PSObject.Properties['requiresNugets'] -and $Metadata.requiresNugets -eq 'true') { $entry['requiresNugets'] = 'true' } if ($Metadata.PSObject.Properties['requiresTestSdk'] -and $Metadata.requiresTestSdk -eq 'true') { $entry['requiresTestSdk'] = 'true' } - if ($Metadata.PSObject.Properties['enablePlaywrightInstall'] -and $Metadata.enablePlaywrightInstall -eq 'true') { $entry['enablePlaywrightInstall'] = 'true' } if ($Metadata.PSObject.Properties['extraTestArgs'] -and $Metadata.extraTestArgs) { $entry['extraTestArgs'] = $Metadata.extraTestArgs } } @@ -147,7 +146,6 @@ function New-CollectionTestEntry { if ($Metadata.PSObject.Properties['requiresNugets'] -and $Metadata.requiresNugets -eq 'true') { $entry['requiresNugets'] = 'true' } if ($Metadata.PSObject.Properties['requiresTestSdk'] -and $Metadata.requiresTestSdk -eq 'true') { $entry['requiresTestSdk'] = 'true' } - if ($Metadata.PSObject.Properties['enablePlaywrightInstall'] -and $Metadata.enablePlaywrightInstall -eq 'true') { $entry['enablePlaywrightInstall'] = 'true' } # Add test filter for collection-based splitting if ($IsUncollected) { @@ -191,7 +189,6 @@ function New-ClassTestEntry { if ($Metadata.PSObject.Properties['testHangTimeout']) { $entry['testHangTimeout'] = $Metadata.testHangTimeout } if ($Metadata.PSObject.Properties['requiresNugets'] -and $Metadata.requiresNugets -eq 'true') { $entry['requiresNugets'] = 'true' } if ($Metadata.PSObject.Properties['requiresTestSdk'] -and $Metadata.requiresTestSdk -eq 'true') { $entry['requiresTestSdk'] = 'true' } - if ($Metadata.PSObject.Properties['enablePlaywrightInstall'] -and $Metadata.enablePlaywrightInstall -eq 'true') { $entry['enablePlaywrightInstall'] = 'true' } # Add test filter for class-based splitting $entry['extraTestArgs'] = "--filter-class `"$ClassName`"" diff --git a/tests/Directory.Build.targets b/tests/Directory.Build.targets index 96455dc3364..eb238744c44 100644 --- a/tests/Directory.Build.targets +++ b/tests/Directory.Build.targets @@ -80,8 +80,7 @@ <_InitialMetadataLines Include=" "testSessionTimeout": "$(TestSessionTimeout)"," /> <_InitialMetadataLines Include=" "testHangTimeout": "$(TestHangTimeout)"," /> <_InitialMetadataLines Include=" "uncollectedTestsSessionTimeout": "$(UncollectedTestsSessionTimeout)"," /> - <_InitialMetadataLines Include=" "uncollectedTestsHangTimeout": "$(UncollectedTestsHangTimeout)"," /> - <_InitialMetadataLines Include=" "enablePlaywrightInstall": "$(EnablePlaywrightInstallForSplitTests)"" /> + <_InitialMetadataLines Include=" "uncollectedTestsHangTimeout": "$(UncollectedTestsHangTimeout)"" /> <_InitialMetadataLines Include="}" /> From 8007d40bcf17bd63b6d4ad0500f16f21a00c1bba Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sat, 18 Oct 2025 01:02:24 -0400 Subject: [PATCH 46/48] checking outerloop space --- .github/workflows/specialized-test-runner.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/specialized-test-runner.yml b/.github/workflows/specialized-test-runner.yml index 2eccb14cf40..d13ce78401a 100644 --- a/.github/workflows/specialized-test-runner.yml +++ b/.github/workflows/specialized-test-runner.yml @@ -36,6 +36,12 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: du -sh + if: ${{ always() }} + run: |- + df -h + du -d 1 -h ${{ github.workspace }} + # We need to build the whole solution, so that we can interrogate each test project # and find out whether it contains any tests of the specified type. - name: Build the solution @@ -61,6 +67,14 @@ jobs: /p:Build=false /bl:${{ github.workspace }}/artifacts/log/Release/runsheet.binlog + - name: du -sh + if: ${{ always() }} + run: |- + df -h + du -d 1 -h ~/.nuget/packages + du -d 1 -h ${{ github.workspace }} + du -d 1 -h ${{ github.workspace }}/artifacts + - name: Upload logs, and test results if: ${{ always() }} uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 From 405158180aa3c35c8fdcd8597270a8ef850e1a15 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sun, 19 Oct 2025 02:54:30 -0400 Subject: [PATCH 47/48] trying a quarantined tests skill --- .claude/skills/quarantine-test/SKILL.md | 228 ++++++++++++++++++ .claude/skills/test-runner/SKILL.md | 166 +++++++++++++ .github/copilot-instructions.md | 142 ++++++++++- .../AddPythonAppTests.cs | 3 +- 4 files changed, 527 insertions(+), 12 deletions(-) create mode 100644 .claude/skills/quarantine-test/SKILL.md create mode 100644 .claude/skills/test-runner/SKILL.md diff --git a/.claude/skills/quarantine-test/SKILL.md b/.claude/skills/quarantine-test/SKILL.md new file mode 100644 index 00000000000..c2f9588d791 --- /dev/null +++ b/.claude/skills/quarantine-test/SKILL.md @@ -0,0 +1,228 @@ +--- +name: Quarantine Test +description: Quarantine flaky or failing tests by adding the QuarantinedTest attribute using the QuarantineTools. Use when tests are failing intermittently or need to be excluded from regular test runs. +allowed-tools: Read, Grep, Bash +--- + +# Quarantine Test + +## Purpose + +This skill quarantines flaky or failing tests by using the QuarantineTools utility to add the `[QuarantinedTest]` attribute. Quarantined tests are excluded from regular test runs and instead run in a separate quarantine workflow. + +## When to Use + +Invoke this skill when: +- A test is failing intermittently (flaky) +- A test doesn't fail deterministically +- User requests to "quarantine a test" +- A test needs to be temporarily excluded from CI while being fixed +- You need to mark a test as quarantined with an associated GitHub issue + +## Important Context + +- Quarantined tests are marked with `[QuarantinedTest("issue-url")]` attribute +- They are NOT run in the regular `tests.yml` workflow +- They ARE run in the separate `tests-quarantine.yml` workflow every 6 hours +- A GitHub issue URL is REQUIRED when quarantining tests +- The QuarantineTools utility handles adding the attribute and managing using directives automatically + +## Instructions + +### Step 1: Identify the Test to Quarantine + +1. Get the fully-qualified test method name in format: `Namespace.ClassName.TestMethodName` +2. If user provides partial name, use Grep to find the complete qualified name: + ```bash + grep -rn "void TestMethodName" tests/ + ``` +3. Extract the namespace and class name from the file to build the fully-qualified name + +Example: `Aspire.Hosting.Tests.DistributedApplicationTests.TestMethodName` + +### Step 2: Get or Create GitHub Issue + +1. Check if the user provided a GitHub issue URL +2. If not, ask: "What is the GitHub issue URL for tracking this flaky test?" +3. The URL must be a valid http/https URL (e.g., `https://github.com/dotnet/aspire/issues/1234`) +4. If no issue exists, suggest creating one first to track the test failure + +### Step 3: Run QuarantineTools + +Execute the QuarantineTools with the quarantine flag, test name(s), and issue URL: + +```bash +dotnet run --project tools/QuarantineTools/QuarantineTools.csproj -- --quarantine "Namespace.ClassName.TestMethodName" --url "https://github.com/org/repo/issues/1234" +``` + +**Multiple tests** can be quarantined at once: +```bash +dotnet run --project tools/QuarantineTools/QuarantineTools.csproj -- --quarantine "Namespace.Class.Test1" "Namespace.Class.Test2" --url "https://github.com/org/repo/issues/1234" +``` + +**Command line flags:** +- `-q` or `--quarantine`: Quarantine mode (add attribute) +- `-i` or `--url`: GitHub issue URL (required for quarantine) +- Tests: Fully-qualified method names (space-separated) + +### Step 4: Verify the Changes + +1. The tool will output which files were modified: + ``` + Updated 1 file(s): + - tests/ProjectName.Tests/TestFile.cs + ``` + +2. Read the modified file to confirm the attribute was added correctly: + ```bash + grep -A 2 -B 2 "QuarantinedTest" tests/ProjectName.Tests/TestFile.cs + ``` + +3. Verify that: + - The `[QuarantinedTest("issue-url")]` attribute appears above the test method + - The `using Aspire.TestUtilities;` directive was added to the file (if not already present) + +### Step 5: Build and Run Tests to Confirm + +1. Build the test project to ensure no compilation errors: + ```bash + dotnet build tests/ProjectName.Tests/ProjectName.Tests.csproj + ``` + +2. Run the test project with quarantine filter to verify the test is now quarantined: + ```bash + dotnet test tests/ProjectName.Tests/ProjectName.Tests.csproj --no-build -- --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true" + ``` + +3. Confirm the quarantined test is NOT executed in the regular test run + +4. Optionally, verify the test CAN be run with the quarantine filter: + ```bash + dotnet test tests/ProjectName.Tests/ProjectName.Tests.csproj --no-build -- --filter-trait "quarantined=true" + ``` + +### Step 6: Report Results + +Provide a clear summary: +- Which test(s) were quarantined +- The GitHub issue URL used +- Which file(s) were modified +- Confirmation that the test builds and is properly excluded from regular runs +- Remind the user to commit the changes + +## Examples + +### Example 1: Quarantine a single flaky test + +User: "Quarantine the TestDistributedApplicationLifecycle test, it's flaky. Issue: https://github.com/dotnet/aspire/issues/5678" + +Actions: +1. Find the fully-qualified name using Grep: + ```bash + grep -rn "void TestDistributedApplicationLifecycle" tests/ + ``` + Result: `tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs` + +2. Determine qualified name: `Aspire.Hosting.Tests.DistributedApplicationTests.TestDistributedApplicationLifecycle` + +3. Run QuarantineTools: + ```bash + dotnet run --project tools/QuarantineTools/QuarantineTools.csproj -- -q "Aspire.Hosting.Tests.DistributedApplicationTests.TestDistributedApplicationLifecycle" -i "https://github.com/dotnet/aspire/issues/5678" + ``` + +4. Verify output shows file was updated + +5. Build and test: + ```bash + dotnet build tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj + dotnet test tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj --no-build -- --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true" + ``` + +6. Report: "Quarantined test `TestDistributedApplicationLifecycle` with issue #5678. The test is now excluded from regular CI runs and will run in the quarantine workflow." + +### Example 2: Quarantine multiple related tests + +User: "These three tests in RedisTests are all flaky, quarantine them: TestRedisConnection, TestRedisCache, TestRedisCommands. Issue: https://github.com/dotnet/aspire/issues/9999" + +Actions: +1. Find fully-qualified names (assuming they're in `Aspire.Components.Tests.RedisTests`) + +2. Run QuarantineTools with multiple tests: + ```bash + dotnet run --project tools/QuarantineTools/QuarantineTools.csproj -- -q "Aspire.Components.Tests.RedisTests.TestRedisConnection" "Aspire.Components.Tests.RedisTests.TestRedisCache" "Aspire.Components.Tests.RedisTests.TestRedisCommands" -i "https://github.com/dotnet/aspire/issues/9999" + ``` + +3. Verify all three were modified + +4. Build and test the project + +5. Report: "Quarantined 3 tests from RedisTests with issue #9999." + +### Example 3: User provides short test name + +User: "Quarantine CanStartDashboard - it keeps timing out" + +Actions: +1. Ask for GitHub issue: "What is the GitHub issue URL for tracking this flaky test?" + +2. User provides: "https://github.com/dotnet/aspire/issues/4321" + +3. Find the test: + ```bash + grep -rn "void CanStartDashboard" tests/ + ``` + Found in: `tests/Aspire.Dashboard.Tests/DashboardTests.cs` + +4. Read the file to determine namespace: `Aspire.Dashboard.Tests` + +5. Build fully-qualified name: `Aspire.Dashboard.Tests.DashboardTests.CanStartDashboard` + +6. Run QuarantineTools: + ```bash + dotnet run --project tools/QuarantineTools/QuarantineTools.csproj -- -q "Aspire.Dashboard.Tests.DashboardTests.CanStartDashboard" -i "https://github.com/dotnet/aspire/issues/4321" + ``` + +7. Build and verify + +8. Report success + +## Common Issues and Troubleshooting + +### Issue: "No method found matching" +- **Cause**: The fully-qualified name is incorrect +- **Solution**: Use Grep to find the exact namespace, class name, and method name + +### Issue: "The test is already quarantined" +- **Cause**: The attribute already exists on the method +- **Solution**: Verify by reading the test file; no action needed + +### Issue: Tool reports "Quarantining requires a valid http(s) URL" +- **Cause**: The issue URL is missing or malformed +- **Solution**: Ensure the URL starts with `http://` or `https://` + +### Issue: Build fails after quarantining +- **Cause**: The QuarantineTools may have encountered a syntax issue (rare) +- **Solution**: Read the modified file and check for syntax errors; the tool should handle this correctly + +## Important Notes + +1. **Always build after quarantining** to ensure the changes are valid +2. **Run tests to confirm** the quarantined test is properly excluded +3. **Don't forget to commit** the modified test files +4. **Track with GitHub issues** - every quarantined test should have an associated issue +5. **The QuarantineTools handles**: + - Adding the `[QuarantinedTest("url")]` attribute + - Adding `using Aspire.TestUtilities;` if needed + - Preserving file formatting and indentation + - Supporting nested classes and various namespace styles + +## Unquarantining Tests + +To unquarantine a test (remove the attribute), use: +```bash +dotnet run --project tools/QuarantineTools/QuarantineTools.csproj -- -u "Namespace.ClassName.TestMethodName" +``` + +The tool will: +- Remove the `[QuarantinedTest]` attribute +- Remove the `using Aspire.TestUtilities;` directive if no other tests in the file use it diff --git a/.claude/skills/test-runner/SKILL.md b/.claude/skills/test-runner/SKILL.md new file mode 100644 index 00000000000..9a10437ecce --- /dev/null +++ b/.claude/skills/test-runner/SKILL.md @@ -0,0 +1,166 @@ +--- +name: Aspire Test Runner +description: Run tests for the Aspire project correctly, excluding quarantined and outerloop tests, with proper build verification. Use when running tests, debugging test failures, or validating changes. +allowed-tools: Read, Grep, Glob, Bash +--- + +# Aspire Test Runner + +## Purpose + +This skill ensures tests are run correctly in the Aspire repository, following the project's specific requirements for test execution, including proper exclusion of quarantined and outerloop tests. + +## When to Use + +Invoke this skill when: +- Running tests for a specific project or test class +- Debugging test failures +- Validating code changes +- Verifying builds after modifications +- User requests to "run tests" or "test my changes" + +## Critical Requirements + +**ALWAYS exclude quarantined and outerloop tests** in automated environments: +- Quarantined tests are marked with `[QuarantinedTest]` and are known to be flaky +- Outerloop tests are marked with `[OuterloopTest]` and are long-running or resource-intensive +- These tests run separately in dedicated CI workflows + +## Instructions + +### Step 1: Identify Test Target + +1. If the user specifies a test project, use that path +2. If the user mentions specific test methods or classes, identify the containing test project +3. Use Glob to find test projects if needed: + ```bash + find tests -name "*.Tests.csproj" -type f + ``` + +### Step 2: Build Verification (if needed) + +**Important**: Only build if: +- There have been code changes since the last build +- The user hasn't just run a successful build +- You're unsure if the code is up to date + +If building is needed: +```bash +# Quick build with skip native (saves 1-2 minutes) +./build.sh --build /p:SkipNativeBuild=true +``` + +### Step 3: Run Tests with Proper Filters + +**Default test run** (all tests in a project): +```bash +dotnet test tests/{ProjectName}.Tests/{ProjectName}.Tests.csproj --no-build -- --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true" +``` + +**Specific test method**: +```bash +dotnet test tests/{ProjectName}.Tests/{ProjectName}.Tests.csproj --no-build -- --filter-method "*.{MethodName}" --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true" +``` + +**Specific test class**: +```bash +dotnet test tests/{ProjectName}.Tests/{ProjectName}.Tests.csproj --no-build -- --filter-class "*.{ClassName}" --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true" +``` + +**Multiple test methods**: +```bash +dotnet test tests/{ProjectName}.Tests/{ProjectName}.Tests.csproj --no-build -- --filter-method "*.Method1" --filter-method "*.Method2" --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true" +``` + +### Step 4: Handle Test Results + +**If tests pass**: +- Report success to the user +- Mention the number of tests that passed + +**If tests fail**: +1. Analyze the failure output +2. Identify which tests failed and why +3. Check if failures are related to recent code changes +4. Suggest fixes or next steps +5. Do NOT mark the task as complete if tests are failing + +**If snapshot tests fail**: +1. Tests using Verify library will show snapshot differences +2. After verifying the new output is correct, run: + ```bash + dotnet verify accept -y + ``` +3. Re-run the tests to confirm they pass + +### Step 5: Report Results + +Provide a clear summary: +- Number of tests run +- Pass/fail status +- Any warnings or issues +- Next steps if failures occurred + +## Examples + +### Example 1: Run all tests for a specific project + +User: "Run tests for Aspire.Hosting.Testing" + +Actions: +1. Identify test project: `tests/Aspire.Hosting.Testing.Tests/Aspire.Hosting.Testing.Tests.csproj` +2. Run with proper filters: + ```bash + dotnet test tests/Aspire.Hosting.Testing.Tests/Aspire.Hosting.Testing.Tests.csproj --no-build -- --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true" + ``` +3. Report results + +### Example 2: Run a specific test method + +User: "Run the TestingBuilderHasAllPropertiesFromRealBuilder test" + +Actions: +1. Find the test using Grep: + ```bash + grep -r "TestingBuilderHasAllPropertiesFromRealBuilder" tests/ + ``` +2. Identify project: `Aspire.Hosting.Testing.Tests` +3. Run specific test: + ```bash + dotnet test tests/Aspire.Hosting.Testing.Tests/Aspire.Hosting.Testing.Tests.csproj --no-build -- --filter-method "*.TestingBuilderHasAllPropertiesFromRealBuilder" --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true" + ``` +4. Report results + +### Example 3: Run tests after making changes + +User: "I just modified the hosting code, run tests to verify" + +Actions: +1. Identify affected test projects (e.g., `Aspire.Hosting.Tests`) +2. Build first since code was modified: + ```bash + ./build.sh --build /p:SkipNativeBuild=true + ``` +3. Run tests: + ```bash + dotnet test tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj -- --filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true" + ``` + Note: No `--no-build` flag since we need to pick up the changes +4. Report results + +## Common Pitfalls to Avoid + +1. **Never omit the quarantine and outerloop filters** - this will run flaky tests +2. **Don't use `--no-build`** if code has changed - the changes won't be tested +3. **Don't run the full test suite** - it takes 30+ minutes, use targeted testing +4. **Don't ignore snapshot test failures** - they indicate output changes that need review +5. **Don't forget the `--` separator** before filter arguments + +## Valid Test Filter Switches + +- `--filter-class` / `--filter-not-class`: Filter by class name +- `--filter-method` / `--filter-not-method`: Filter by method name +- `--filter-namespace` / `--filter-not-namespace`: Filter by namespace +- `--filter-trait` / `--filter-not-trait`: Filter by trait (category, platform, etc.) + +Switches can be repeated to filter multiple values. Class and method filters expect fully qualified names, unless using a prefix like `*.ClassName`. diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 35e8eb89da5..9a1dfb516e2 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -156,17 +156,6 @@ These switches can be repeated to run tests on multiple classes or methods at on 2. **Package Generation**: `./build.sh --pack` verifies all packages can be created 3. **Specific Tests**: Target individual test projects related to your changes -## Quarantined tests - -- Tests that are flaky and don't fail deterministically are marked with the `QuarantinedTest` attribute. -- Such tests are not run as part of the regular tests workflow (`tests.yml`). - - Instead they are run in the `Quarantine` workflow (`tests-quarantine.yml`). -- A github issue url is used with the attribute - -Example: `[QuarantinedTest("..issue url..")]` - -- To quarantine or unquarantine tests, use the tool in `tools/QuarantineTools/QuarantineTools.csproj`. - ## Outerloop tests - Tests that are long-running, resource-intensive, or require special infrastructure are marked with the `OuterloopTest` attribute. @@ -194,6 +183,137 @@ The `*.Designer.cs` files are in the repo, but are intended to match same named * Code blocks should be formatted with triple backticks (```) and include the language identifier for syntax highlighting. * JSON code blocks should be indented properly. +## Custom Skills System + +This repository supports custom skills - reusable, specialized capabilities that extend Copilot's functionality for specific tasks. Skills work like plugins that you should automatically invoke when relevant to the user's request. + +### Skill Locations + +Skills are stored in the following directories (in order of precedence): +1. **Project Skills** (shared with team): `.claude/skills/` - checked into git +2. **Personal Skills** (user-specific): `~/.claude/skills/` - not checked into git + +### Skill Structure + +Each skill is a directory containing a `SKILL.md` file with this format: + +```markdown +--- +name: Human Readable Name (max 64 chars) +description: One-line description of what the skill does and when to use it (max 1024 chars) +allowed-tools: Read, Grep, Glob, Bash, Edit, Write # Optional: restrict which tools the skill can use +--- + +# Skill Name + +## Purpose +Clear explanation of what this skill does. + +## When to Use +Specific scenarios when this skill should be invoked. + +## Instructions +Step-by-step guidance for executing the skill. + +## Examples +Concrete examples demonstrating the skill's usage. +``` + +### How to Discover and Use Skills + +**IMPORTANT**: You must proactively check for and use skills when they match the user's request. + +#### Step 1: Skill Discovery + +When a user makes a request, before starting work: + +1. **Search for relevant skills** by checking `.claude/skills/` and `~/.claude/skills/`: + ```bash + # List all available skills + find .claude/skills -name "SKILL.md" -type f 2>/dev/null + find ~/.claude/skills -name "SKILL.md" -type f 2>/dev/null + ``` + +2. **Read skill descriptions** to find matches: + - Read the YAML frontmatter of each `SKILL.md` file + - Match the `description` field against the user's request + - A skill is relevant if its description keywords overlap with the task + +#### Step 2: Skill Invocation + +When you identify a relevant skill: + +1. **Announce skill activation** to the user: + ``` + I found a relevant skill: "{skill_name}". I'll use this to help with your request. + ``` + +2. **Read the complete SKILL.md file** to understand the full instructions: + - Use the Read tool to load the entire skill file + - Parse both the frontmatter and the markdown content + - Pay special attention to the Instructions and Examples sections + +3. **Follow the skill's instructions exactly**: + - Execute each step in the Instructions section + - Use only the tools specified in `allowed-tools` (if defined) + - Adapt the examples to the current context + - If the skill references other files in its directory, read them as needed + +4. **Report completion**: + ``` + Completed task using the "{skill_name}" skill. + ``` + +#### Step 3: When to Use Subagents for Skills + +For complex skills that involve multiple phases or extensive codebase exploration, use subagents: + +1. **Use Task tool with subagent_type=Explore** when a skill requires: + - Searching across multiple files or directories + - Understanding codebase patterns before making changes + - Gathering context from various locations + +2. **Use Task tool with subagent_type=general-purpose** when a skill requires: + - Multi-step workflows that could fail and need retry logic + - Complex decision trees based on what's found in the codebase + - Long-running operations that benefit from isolation + +Example of using a subagent for skill execution: +``` +I'll use the Task tool to invoke the "{skill_name}" skill: +[Invoke Task tool with subagent_type=Explore and pass the full skill instructions as the prompt] +``` + +### Skill Best Practices + +1. **Always check for skills first** before starting any non-trivial task +2. **Trust the skill instructions** - they are curated and tested +3. **Don't modify skills** unless explicitly asked by the user +4. **Combine skills** when multiple skills apply to different parts of a task +5. **Report when no skill exists** - if a task would benefit from a skill but none exists, mention this to the user + +### Example: Using a Skill + +User request: "Review the code changes in my PR for performance issues" + +Your workflow: +1. Search for skills: `find .claude/skills -name "SKILL.md"` +2. Find match: `.claude/skills/performance-reviewer/SKILL.md` +3. Announce: "I found the 'Performance Reviewer' skill. I'll use this to analyze your PR." +4. Read skill: Load `.claude/skills/performance-reviewer/SKILL.md` +5. Execute: Follow the skill's instructions step-by-step +6. Complete: "Completed performance review using the Performance Reviewer skill." + +### Creating New Skills (When Asked) + +If a user asks you to create a new skill: + +1. Create a directory in `.claude/skills/{skill-name}/` +2. Write a `SKILL.md` file following the structure above +3. Include clear, actionable instructions +4. Add concrete examples +5. Test the skill by using it immediately + ## Trust These Instructions These instructions are comprehensive and tested. Only search for additional information if: diff --git a/tests/Aspire.Hosting.Python.Tests/AddPythonAppTests.cs b/tests/Aspire.Hosting.Python.Tests/AddPythonAppTests.cs index c5b76246e67..3a55d2637dd 100644 --- a/tests/Aspire.Hosting.Python.Tests/AddPythonAppTests.cs +++ b/tests/Aspire.Hosting.Python.Tests/AddPythonAppTests.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. #pragma warning disable CS0612 @@ -476,6 +476,7 @@ public void WithVirtualEnvironment_ThrowsOnNullOrEmptyPath() } [Fact] + [QuarantinedTest("https://foo.com/issue/123")] public async Task WithVirtualEnvironment_CanBeChainedWithOtherExtensions() { using var builder = TestDistributedApplicationBuilder.Create().WithTestAndResourceLogging(outputHelper); From 531c9244b7a12aa02176c7c6558502b0832b3d07 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sun, 19 Oct 2025 03:02:44 -0400 Subject: [PATCH 48/48] remove md --- .github/instructions/quarantine.instructions.md | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 .github/instructions/quarantine.instructions.md diff --git a/.github/instructions/quarantine.instructions.md b/.github/instructions/quarantine.instructions.md deleted file mode 100644 index 4d4952350ce..00000000000 --- a/.github/instructions/quarantine.instructions.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -applyTo: "tools/QuarantineTools/*" ---- - -This tool is used to quarantine flaky tests. - -Usage: - -```bash -dotnet run --project tools/QuarantineTools -- -q Namespace.Type.Method -i https://issue.url -``` - -Make sure to build the project containing the updated tests to ensure the changes don't break the build.