WinUI Runtime Tests

build/ci/tests/.azure-devops-tests-winappsdk.yml

@@ -1,5 +1,5 @@
 parameters:
-  vmImage: ''
+  poolName: ''
 
 jobs:
 - job: WASDK_Build
@@ -29,7 +29,16 @@ jobs:
   # Required until Uno.Xaml can build without all targets enabled
   - template: ../templates/jdk-setup.yml
 
+  - pwsh: |
+      # Decode the base64-encoded signing certificate from the variable
+      $certBytes = [Convert]::FromBase64String('$(WINAPPSDK_SAMPLESAPP_SIGNING_CERTIFICATE)')
+      $certPath = "$(Build.SourcesDirectory)/src/SamplesApp/SamplesApp.Windows/SamplesApp.Windows_TemporaryKey.pfx"
+      [IO.File]::WriteAllBytes($certPath, $certBytes)
+      Write-Host "Certificate written to: $certPath"
+    displayName: Decode Signing Certificate
+
   - task: MSBuild@1
+    name: BuildMSIX
     inputs:
       solution: src\SamplesApp\SamplesApp.Windows\SamplesApp.Windows.csproj
       msbuildLocationMethod: version
@@ -43,13 +52,24 @@ jobs:
       # ---
       # NOTE: The error says to specify a RuntimeIdentifier *OR* platform other than AnyCPU.
       # We already specify RuntimeIdentifier=win-x64 in the build below. Still, the error pops up.
-      msbuildArguments: /r /t:Publish /m /v:m /p:Configuration=Release /p:Platform=x64 /p:RuntimeIdentifier=win-x64 /p:BuildGraphics3DGLForWindows=true /p:GenerateAppxPackageOnBuild=true /detailedsummary /bl:$(build.artifactstagingdirectory)/build-wasdk.binlog
+      msbuildArguments: /r /t:Publish /m /v:m /p:Configuration=Release /p:Platform=x64 /p:RuntimeIdentifier=win-x64 /p:BuildGraphics3DGLForWindows=true /p:GenerateAppxPackageOnBuild=true /p:PackageCertificateKeyFile=$(Build.SourcesDirectory)/src/SamplesApp/SamplesApp.Windows/SamplesApp.Windows_TemporaryKey.pfx /detailedsummary /bl:$(build.artifactstagingdirectory)/build-wasdk.binlog
       clean: false
       restoreNugetPackages: false
       logProjectEvents: false
       createLogFile: false
 
+  - pwsh: |
+      # Clean up the build-time signing certificate
+      $certPath = "$(Build.SourcesDirectory)/src/SamplesApp/SamplesApp.Windows/SamplesApp.Windows_TemporaryKey.pfx"
+      if (Test-Path $certPath) {
+        Remove-Item $certPath -Force
+        Write-Host "Build signing certificate cleaned up"
+      }
+    displayName: Clean Up Build Signing Certificate
+    condition: always()
+
   - task: CopyFiles@2
+    displayName: 'Copy MSIX Packages'
     condition: always()
     inputs:
       SourceFolder: $(build.sourcesdirectory)/src/SamplesApp/SamplesApp.Windows/AppPackages
@@ -66,3 +86,117 @@ jobs:
       PathtoPublish: $(build.artifactstagingdirectory)
       ArtifactName: WinAppSDK-Package
       ArtifactType: Container
+
+##
+## WinUI Runtime Tests
+##
+## Note: If tests fail on WinUI, they should be excluded using the [PlatformCondition] attribute
+## in the test code itself. Example:
+##
+##   [TestMethod]
+##   [PlatformCondition(ConditionMode.Exclude, RuntimeTestPlatforms.NativeWinUI)]
+##   public void When_Test_That_Fails_On_WinUI() { ... }
+##
+## See: doc/articles/uno-development/creating-runtime-tests.md
+##
+- job: WinUI_Runtime_Tests
+  displayName: 'Runtime Tests'
+  dependsOn: WASDK_Build
+  timeoutInMinutes: 30
+  cancelTimeoutInMinutes: 1
+
+  pool:
+    vmImage: 'windows-latest'
+
+  variables:
+    SamplesAppArtifactName: WinAppSDK-Package
+    SamplesAppArtifactPath: $(build.sourcesdirectory)/build/$(SamplesAppArtifactName)
+
+  steps:
+  - checkout: self
+    clean: true
+
+  - task: DownloadPipelineArtifact@2
+    displayName: Downloading $(SamplesAppArtifactName)
+    inputs:
+      artifact: $(SamplesAppArtifactName)
+      path: $(build.sourcesdirectory)/build/$(SamplesAppArtifactName)
+
+  - task: DownloadBuildArtifacts@0
+    condition: gt(variables['System.JobAttempt'], 1)
+    continueOnError: true
+    displayName: Download previous test runs failed tests
+    inputs:
+        artifactName: uitests-failure-results
+        downloadPath: '$(build.sourcesdirectory)/build'
+
+  - template: ../templates/dotnet-install.yml
+
+  - pwsh: |
+      # Install Windows App Runtime 1.8
+      Write-Host "Installing Windows App Runtime 1.8..."
+      $runtimeUrl = "https://aka.ms/windowsappsdk/1.8/latest/windowsappruntimeinstall-x64.exe"
+      $runtimeInstaller = "$(Build.SourcesDirectory)/build/windowsappruntimeinstall-x64.exe"
+
+      Write-Host "Downloading from: $runtimeUrl"
+      Invoke-WebRequest -Uri $runtimeUrl -OutFile $runtimeInstaller
+
+      Write-Host "Running installer: $runtimeInstaller"
+      Start-Process -FilePath $runtimeInstaller -ArgumentList "--quiet" -Wait -NoNewWindow
+      Write-Host "Windows App Runtime 1.8 installed successfully"
+    displayName: Install Windows App Runtime
+
+  - pwsh: |
+      # Decode and import the signing certificate to trusted stores
+      $certBytes = [Convert]::FromBase64String('$(WINAPPSDK_SAMPLESAPP_SIGNING_CERTIFICATE)')
+      $certPath = "$(Build.SourcesDirectory)/build/SamplesApp_SigningCert.pfx"
+      try {
+        [IO.File]::WriteAllBytes($certPath, $certBytes)
+
+        # Import the certificate to LocalMachine\Root (Trusted Root CAs) for MSIX installation
+        Import-PfxCertificate -FilePath $certPath -CertStoreLocation Cert:\LocalMachine\Root -ErrorAction Stop
+        Write-Host "Certificate imported to Trusted Root Certification Authorities store"
+      }
+      finally {
+        if (Test-Path $certPath) {
+          Remove-Item $certPath -Force
+          Write-Host "Certificate file cleaned up"
+        }
+      }
+
+      # Find and install the MSIX bundle
+      $msixBundlePath = Get-ChildItem -Path "$(SamplesAppArtifactPath)/AppPackages" -Filter "*.msixbundle" -Recurse | Select-Object -First 1
+      if (-not $msixBundlePath) {
+        throw "Could not find MSIX bundle in $(SamplesAppArtifactPath)/AppPackages"
+      }
+
+      Write-Host "Installing MSIX bundle: $($msixBundlePath.FullName)"
+      Add-AppxPackage -Path $msixBundlePath.FullName
+    displayName: Install MSIX Package
+
+  - pwsh: build/test-scripts/run-winui-runtime-tests.ps1
+    displayName: Run WinUI Runtime Tests
+
+  - pwsh: |
+      # Uninstall the MSIX package
+      Get-AppxPackage -Name "SamplesApp" | Remove-AppxPackage -ErrorAction SilentlyContinue
+    displayName: Uninstall MSIX Package
+    condition: always()
+
+  - task: PublishTestResults@2
+    condition: always()
+    inputs:
+      testRunTitle: 'WinUI Runtime Tests'
+      testResultsFormat: 'NUnit'
+      testResultsFiles: '$(build.sourcesdirectory)/build/winui-runtime-tests-results.xml'
+      failTaskOnFailedTests: true
+      failTaskOnMissingResultsFile: true
+
+  - task: PublishBuildArtifacts@1
+    condition: always()
+    displayName: Publish Failed Tests Results
+    retryCountOnTaskFailure: 3
+    inputs:
+      PathtoPublish: $(build.sourcesdirectory)/build/uitests-failure-results
+      ArtifactName: uitests-failure-results
+      ArtifactType: Container

build/test-scripts/run-winui-runtime-tests.ps1

@@ -0,0 +1,117 @@
+$ErrorActionPreference = 'Stop'
+
+function Assert-ExitCodeIsZero()
+{
+    if ($LASTEXITCODE -ne 0)
+    {
+        throw "Exit code must be zero."
+	}
+}
+
+$UNO_TESTS_FAILED_LIST="$env:BUILD_SOURCESDIRECTORY\build\uitests-failure-results\failed-tests-winui-runtimetests.txt"
+$TEST_RESULTS_FILE="$env:BUILD_SOURCESDIRECTORY\build\winui-runtime-tests-results.xml"
+
+# convert the content of the file UNO_TESTS_FAILED_LIST to base64 and set it to UITEST_RUNTIME_TESTS_FILTER, if the file exists
+if (Test-Path $UNO_TESTS_FAILED_LIST) {
+    $base64 = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes((Get-Content -Raw $UNO_TESTS_FAILED_LIST)))
+    $env:UITEST_RUNTIME_TESTS_FILTER="$base64"
+}
+
+# Build command-line arguments for runtime tests
+$runtimeTestArgs = @("--runtime-tests=`"$TEST_RESULTS_FILE`"")
+
+if ($env:UITEST_RUNTIME_TEST_GROUP) {
+    $runtimeTestArgs += "--runtime-tests-group=$env:UITEST_RUNTIME_TEST_GROUP"
+}
+
+if ($env:UITEST_RUNTIME_TEST_GROUP_COUNT) {
+    $runtimeTestArgs += "--runtime-tests-group-count=$env:UITEST_RUNTIME_TEST_GROUP_COUNT"
+}
+
+if ($env:UITEST_RUNTIME_TESTS_FILTER) {
+    $runtimeTestArgs += "--runtime-test-filter=$env:UITEST_RUNTIME_TESTS_FILTER"
+}
+
+Write-Host "Runtime test arguments: $($runtimeTestArgs -join ' ')"
+
+# Ensure the failed tests directory exists before running tests
+$failedTestsDir = Split-Path -Parent $UNO_TESTS_FAILED_LIST
+New-Item -ItemType Directory -Force -Path $failedTestsDir | Out-Null
+
+# Use the app execution alias registered by the MSIX package
+$exeAlias = "unosamplesapp.exe"
+
+Write-Host "Using app execution alias: $exeAlias"
+
+# Launch the app with runtime test arguments
+Write-Host "Launching app with runtime tests..."
+
+$process = Start-Process -FilePath $exeAlias -ArgumentList $runtimeTestArgs -PassThru -NoNewWindow
+
+Write-Host "App launched with PID: $($process.Id)"
+Write-Host "Waiting for test results..."
+
+# Wait for the test results file with timeout
+$timeout = 600 # 10 minutes
+$elapsed = 0
+$checkInterval = 5
+
+while ($elapsed -lt $timeout) {
+    # Check if process has exited
+    if ($process.HasExited) {
+        Write-Host "App process has exited with code: $($process.ExitCode)"
+        break
+    }
+
+    if (Test-Path $TEST_RESULTS_FILE) {
+        Write-Host "Test results file found!"
+        # Wait a bit more for the app to finish writing and exit cleanly
+        Start-Sleep -Seconds 5
+        break
+    }
+
+    Start-Sleep -Seconds $checkInterval
+    $elapsed += $checkInterval
+    Write-Host "Waiting for test results... ($elapsed seconds elapsed)"
+}
+
+if (-not (Test-Path $TEST_RESULTS_FILE)) {
+    Write-Host "Test results file was not created within the timeout period"
+
+    # Try to get the app process logs if still running
+    if (-not $process.HasExited) {
+        Write-Host "Force stopping the app..."
+        Stop-Process -Id $process.Id -Force -ErrorAction SilentlyContinue
+    }
+
+    throw "Test results file was not created"
+}
+
+# Ensure the process has exited
+if (-not $process.HasExited) {
+    Write-Host "Waiting for app to exit gracefully..."
+    $process.WaitForExit(30000) # Wait up to 30 seconds
+
+    if (-not $process.HasExited) {
+        Write-Host "Force stopping the app..."
+        Stop-Process -Id $process.Id -Force -ErrorAction SilentlyContinue
+    }
+}
+
+Write-Host "Test execution completed. Processing results..."
+
+## Export the failed tests list for reuse in a pipeline retry
+Push-Location "$env:BUILD_SOURCESDIRECTORY/src/Uno.NUnitTransformTool"
+
+Write-Host "Running NUnitTransformTool"
+
+## Fail the build when no test results could be read
+dotnet run fail-empty $TEST_RESULTS_FILE
+
+Assert-ExitCodeIsZero
+
+dotnet run list-failed $TEST_RESULTS_FILE $UNO_TESTS_FAILED_LIST
+
+Assert-ExitCodeIsZero
+
+Pop-Location

doc/articles/uno-development/creating-runtime-tests.md

@@ -30,6 +30,59 @@ To add a new runtime test:
 4. Use the helpers described below to set up the state required to repro your bug, and use standard `Assert` calls and/or [FluentAssertions](https://fluentassertions.com/introduction) to verify expected state.
 5. Verify your test locally by running according to the instructions above. Typically it's fine to test locally on just one target platform; the CI will take care of the others.
 
+### Platform-specific test exclusions
+
+Runtime tests can be conditionally excluded or included for specific platforms using the `[PlatformCondition]` attribute. This is useful when a feature is not yet implemented or behaves differently on certain platforms.
+
+#### Using PlatformCondition
+
+The `PlatformConditionAttribute` allows you to specify which platforms a test should run on or be excluded from:
+
+```csharp
+// Exclude test from WinUI
+[TestMethod]
+[PlatformCondition(ConditionMode.Exclude, RuntimeTestPlatforms.NativeWinUI)]
+public void When_Test_That_Fails_On_WinUI()
+{
+    // Test implementation
+}
+
+// Run test only on specific platforms
+[TestMethod]
+[PlatformCondition(ConditionMode.Include, RuntimeTestPlatforms.Wasm | RuntimeTestPlatforms.Skia)]
+public void When_Test_Only_For_Wasm_And_Skia()
+{
+    // Test implementation
+}
+
+// Exclude from multiple platforms
+[TestMethod]
+[PlatformCondition(ConditionMode.Exclude, RuntimeTestPlatforms.NativeAndroid | RuntimeTestPlatforms.NativeIOS)]
+public void When_Test_Not_Supported_On_Mobile()
+{
+    // Test implementation
+}
+```
+
+#### Available platforms
+
+The `RuntimeTestPlatforms` enum includes the following platforms:
+
+- **Native platforms**: `NativeWinUI`, `NativeWasm`, `NativeAndroid`, `NativeIOS`, `NativeMacCatalyst`, `NativeTvOS`
+- **Skia platforms**: `SkiaWpf`, `SkiaWin32`, `SkiaX11`, `SkiaMacOS`, `SkiaIslands`, `SkiaWasm`, `SkiaAndroid`, `SkiaIOS`, `SkiaMacCatalyst`, `SkiaTvOS`, `SkiaFrameBuffer`
+- **Combined platforms**:
+  - `NativeUIKit` (NativeIOS | NativeTvOS | NativeMacCatalyst)
+  - `SkiaUIKit` (SkiaIOS | SkiaTvOS | SkiaMacCatalyst)
+  - `SkiaMobile` (SkiaAndroid | SkiaUIKit)
+  - `SkiaDesktop` (SkiaWpf | SkiaWin32 | SkiaX11 | SkiaMacOS | SkiaIslands | SkiaFrameBuffer)
+  - `Skia` (SkiaDesktop | SkiaWasm | SkiaMobile)
+  - `Native` (NativeWasm | NativeAndroid | NativeIOS | NativeMacCatalyst | NativeTvOS | NativeWinUI)
+  - `Wasm` (NativeWasm | SkiaWasm)
+  - `Android` (NativeAndroid | SkiaAndroid)
+  - `IOS` (NativeIOS | SkiaIOS)
+
+When using platform exclusions, always add a comment or issue reference explaining why the test is excluded, to help track and resolve platform-specific issues.
+
 ### Helpers
 
 A number of helper methods are available to set up a state for testing common UI scenarios.

src/SamplesApp/SamplesApp.Shared/App.xaml.cs

@@ -90,6 +90,9 @@ static App()
 		/// </summary>
 		public App()
 		{
+#if !HAS_UNO
+			UnhandledException += App_UnhandledException;
+#endif
 			// Fix language for UI tests
 			Thread.CurrentThread.CurrentCulture = new CultureInfo("en-US");
 			Thread.CurrentThread.CurrentUICulture = new CultureInfo("en-US");
@@ -118,6 +121,19 @@ public App()
 #endif
 		}
 
+#if !HAS_UNO
+		private void App_UnhandledException(object sender, Microsoft.UI.Xaml.UnhandledExceptionEventArgs e)
+		{
+			_log?.Error("UnhandledException", e.Exception);
+			global::System.Console.WriteLine($"UnhandledException: {e.Exception}");
+
+			if (Debugger.IsAttached)
+			{
+				e.Handled = true;
+			}
+		}
+#endif
+
 		internal static Microsoft.UI.Xaml.Window? MainWindow => _mainWindow;
 
 		/// <summary>
@@ -391,9 +407,13 @@ private void InitializeFrame(string? arguments = null)
 
 		private async void HandleLaunchArguments(LaunchActivatedEventArgs launchActivatedEventArgs)
 		{
-			Console.WriteLine($"HandleLaunchArguments: {launchActivatedEventArgs.Arguments}");
-
+#if !HAS_UNO
+			var args = string.Join("&", Environment.GetCommandLineArgs().Skip(1));
+#else
 			var args = launchActivatedEventArgs.Arguments ?? "";
+#endif
+
+			Console.WriteLine($"HandleLaunchArguments: {args}");
 
 			if (HandleAutoScreenshots(args))
 			{