diff --git a/build/RunTestsOnHelix.cmd b/build/RunTestsOnHelix.cmd index 569607f04333..9e29fa829a20 100644 --- a/build/RunTestsOnHelix.cmd +++ b/build/RunTestsOnHelix.cmd @@ -9,6 +9,10 @@ set DOTNET_ROOT=%HELIX_CORRELATION_PAYLOAD%\d set PATH=%DOTNET_ROOT%;%PATH% set TestFullMSBuild=%1 +REM Set DOTNET_HOST_PATH so MSBuild task hosts can locate the dotnet executable. +REM Without this, tasks from NuGet packages that use TaskHostFactory fail with MSB4216. +set DOTNET_HOST_PATH=%DOTNET_ROOT%\dotnet.exe + REM Ensure Visual Studio instances allow preview SDKs PowerShell -ExecutionPolicy ByPass -NoProfile -File "%HELIX_CORRELATION_PAYLOAD%\t\eng\enable-preview-sdks.ps1" @@ -35,14 +39,16 @@ dotnet new --debug:ephemeral-hive dotnet nuget list source --configfile %TestExecutionDirectory%\nuget.config if exist %TestExecutionDirectory%\Testpackages dotnet nuget add source %TestExecutionDirectory%\Testpackages --name testpackages --configfile %TestExecutionDirectory%\nuget.config -dotnet nuget remove source dotnet6-transport --configfile %TestExecutionDirectory%\nuget.config -dotnet nuget remove source dotnet6-internal-transport --configfile %TestExecutionDirectory%\nuget.config -dotnet nuget remove source dotnet7-transport --configfile %TestExecutionDirectory%\nuget.config -dotnet nuget remove source dotnet7-internal-transport --configfile %TestExecutionDirectory%\nuget.config -dotnet nuget remove source richnav --configfile %TestExecutionDirectory%\nuget.config -dotnet nuget remove source vs-impl --configfile %TestExecutionDirectory%\nuget.config -dotnet nuget remove source dotnet-libraries-transport --configfile %TestExecutionDirectory%\nuget.config -dotnet nuget remove source dotnet-tools-transport --configfile %TestExecutionDirectory%\nuget.config -dotnet nuget remove source dotnet-libraries --configfile %TestExecutionDirectory%\nuget.config -dotnet nuget remove source dotnet-eng --configfile %TestExecutionDirectory%\nuget.config +REM Remove feeds not needed for tests. Errors from non-existent sources +REM (e.g. internal-transport feeds only present in internal builds) are ignored. +dotnet nuget remove source dotnet6-transport --configfile %TestExecutionDirectory%\nuget.config 2>nul +dotnet nuget remove source dotnet6-internal-transport --configfile %TestExecutionDirectory%\nuget.config 2>nul +dotnet nuget remove source dotnet7-transport --configfile %TestExecutionDirectory%\nuget.config 2>nul +dotnet nuget remove source dotnet7-internal-transport --configfile %TestExecutionDirectory%\nuget.config 2>nul +dotnet nuget remove source richnav --configfile %TestExecutionDirectory%\nuget.config 2>nul +dotnet nuget remove source vs-impl --configfile %TestExecutionDirectory%\nuget.config 2>nul +dotnet nuget remove source dotnet-libraries-transport --configfile %TestExecutionDirectory%\nuget.config 2>nul +dotnet nuget remove source dotnet-tools-transport --configfile %TestExecutionDirectory%\nuget.config 2>nul +dotnet nuget remove source dotnet-libraries --configfile %TestExecutionDirectory%\nuget.config 2>nul +dotnet nuget remove source dotnet-eng --configfile %TestExecutionDirectory%\nuget.config 2>nul dotnet nuget list source --configfile %TestExecutionDirectory%\nuget.config diff --git a/build/RunTestsOnHelix.sh b/build/RunTestsOnHelix.sh index 887748f918b4..973eab473c0e 100644 --- a/build/RunTestsOnHelix.sh +++ b/build/RunTestsOnHelix.sh @@ -9,6 +9,12 @@ export MicrosoftNETBuildExtensionsTargets=$HELIX_CORRELATION_PAYLOAD/ex/msbuildE export DOTNET_ROOT=$HELIX_CORRELATION_PAYLOAD/d export PATH=$DOTNET_ROOT:$PATH +# Set DOTNET_HOST_PATH so MSBuild task hosts can locate the dotnet executable. +# Without this, tasks from NuGet packages that use TaskHostFactory (e.g. ComputeWasmBuildAssets +# from WebAssembly SDK, ComputeManagedAssemblies from ILLink) fail with MSB4216 on macOS +# because the task host process cannot find the dotnet host to launch. +export DOTNET_HOST_PATH=$DOTNET_ROOT/dotnet + export TestExecutionDirectory=$(realpath "$(mktemp -d "${TMPDIR:-/tmp}"/dotnetSdkTests.XXXXXXXX)") export DOTNET_CLI_HOME=$TestExecutionDirectory/.dotnet cp -a $HELIX_CORRELATION_PAYLOAD/t/TestExecutionDirectoryFiles/. $TestExecutionDirectory/ @@ -22,15 +28,17 @@ dotnet new --debug:ephemeral-hive dotnet nuget list source --configfile $TestExecutionDirectory/NuGet.config dotnet nuget add source $TestExecutionDirectory/Testpackages --configfile $TestExecutionDirectory/NuGet.config -#Remove feeds not needed for tests -dotnet nuget remove source dotnet6-transport --configfile $TestExecutionDirectory/NuGet.config -dotnet nuget remove source dotnet6-internal-transport --configfile $TestExecutionDirectory/NuGet.config -dotnet nuget remove source dotnet7-transport --configfile $TestExecutionDirectory/NuGet.config -dotnet nuget remove source dotnet7-internal-transport --configfile $TestExecutionDirectory/NuGet.config -dotnet nuget remove source richnav --configfile $TestExecutionDirectory/NuGet.config -dotnet nuget remove source vs-impl --configfile $TestExecutionDirectory/NuGet.config -dotnet nuget remove source dotnet-libraries-transport --configfile $TestExecutionDirectory/NuGet.config -dotnet nuget remove source dotnet-tools-transport --configfile $TestExecutionDirectory/NuGet.config -dotnet nuget remove source dotnet-libraries --configfile $TestExecutionDirectory/NuGet.config -dotnet nuget remove source dotnet-eng --configfile $TestExecutionDirectory/NuGet.config +# Remove feeds not needed for tests. Use || true to avoid errors when a source +# doesn't exist (e.g. internal-transport feeds are only present in internal builds). +dotnet nuget remove source dotnet6-transport --configfile $TestExecutionDirectory/NuGet.config || true +dotnet nuget remove source dotnet6-internal-transport --configfile $TestExecutionDirectory/NuGet.config || true +dotnet nuget remove source dotnet7-transport --configfile $TestExecutionDirectory/NuGet.config || true +dotnet nuget remove source dotnet7-internal-transport --configfile $TestExecutionDirectory/NuGet.config || true +dotnet nuget remove source richnav --configfile $TestExecutionDirectory/NuGet.config || true +dotnet nuget remove source vs-impl --configfile $TestExecutionDirectory/NuGet.config || true +dotnet nuget remove source dotnet-libraries-transport --configfile $TestExecutionDirectory/NuGet.config || true +dotnet nuget remove source dotnet-tools-transport --configfile $TestExecutionDirectory/NuGet.config || true +dotnet nuget remove source dotnet-libraries --configfile $TestExecutionDirectory/NuGet.config || true +dotnet nuget remove source dotnet-eng --configfile $TestExecutionDirectory/NuGet.config || true dotnet nuget list source --configfile $TestExecutionDirectory/NuGet.config + diff --git a/src/BlazorWasmSdk/Tasks/GZipCompress.cs b/src/BlazorWasmSdk/Tasks/GZipCompress.cs index 96481d04a91b..b5edfd894c14 100644 --- a/src/BlazorWasmSdk/Tasks/GZipCompress.cs +++ b/src/BlazorWasmSdk/Tasks/GZipCompress.cs @@ -20,6 +20,10 @@ public class GZipCompress : Task [Required] public string OutputDirectory { get; set; } + // Retry count for transient file I/O errors (e.g., antivirus locks on CI machines). + private const int MaxRetries = 3; + private const int RetryDelayMs = 200; + public override bool Execute() { CompressedFiles = new ITaskItem[FilesToCompress.Length]; @@ -56,18 +60,31 @@ public override bool Execute() Log.LogMessage(MessageImportance.Low, "Compressing '{0}' because file is newer than '{1}'.", inputFullPath, outputRelativePath); } - try + // Retry on IOException to handle transient file locks from antivirus, file + // indexing, or parallel MSBuild nodes on CI machines (see dotnet/sdk#53424). + for (int attempt = 1; attempt <= MaxRetries; attempt++) { - using var sourceStream = File.OpenRead(file.ItemSpec); - using var fileStream = File.Create(outputRelativePath); - using var stream = new GZipStream(fileStream, CompressionLevel.Optimal); + try + { + using var sourceStream = File.OpenRead(file.ItemSpec); + using var fileStream = File.Create(outputRelativePath); + using var stream = new GZipStream(fileStream, CompressionLevel.Optimal); - sourceStream.CopyTo(stream); - } - catch (Exception e) - { - Log.LogErrorFromException(e); - return; + sourceStream.CopyTo(stream); + return; // Success + } + catch (IOException) when (attempt < MaxRetries) + { + Log.LogMessage(MessageImportance.Low, + "Retrying compression of '{0}' (attempt {1}/{2}) due to transient I/O error.", + file.ItemSpec, attempt, MaxRetries); + Thread.Sleep(RetryDelayMs * attempt); + } + catch (Exception e) + { + Log.LogErrorFromException(e); + return; + } } }); diff --git a/src/Dotnet.Watch/Watch/UI/PhysicalConsole.cs b/src/Dotnet.Watch/Watch/UI/PhysicalConsole.cs index c835611b5d72..2df209b3609a 100644 --- a/src/Dotnet.Watch/Watch/UI/PhysicalConsole.cs +++ b/src/Dotnet.Watch/Watch/UI/PhysicalConsole.cs @@ -13,7 +13,15 @@ internal sealed class PhysicalConsole : IConsole public PhysicalConsole(TestFlags testFlags) { Console.OutputEncoding = Encoding.UTF8; - _ = testFlags.HasFlag(TestFlags.ReadKeyFromStdin) ? ListenToStandardInputAsync() : ListenToConsoleKeyPressAsync(); + + if (testFlags.HasFlag(TestFlags.ReadKeyFromStdin)) + { + _ = ListenToStandardInputAsync(); + } + else if (!Console.IsInputRedirected) + { + _ = ListenToConsoleKeyPressAsync(); + } } private async Task ListenToStandardInputAsync() diff --git a/test/Microsoft.DotNet.HotReload.Test.Utilities/AwaitableProcess.cs b/test/Microsoft.DotNet.HotReload.Test.Utilities/AwaitableProcess.cs index 756897145d2b..c15842c02288 100644 --- a/test/Microsoft.DotNet.HotReload.Test.Utilities/AwaitableProcess.cs +++ b/test/Microsoft.DotNet.HotReload.Test.Utilities/AwaitableProcess.cs @@ -205,6 +205,17 @@ public async ValueTask DisposeAsync() Process.ErrorDataReceived -= OnData; Process.OutputDataReceived -= OnData; + // Close stdin before killing the process to unblock any pending stdin reads + // (e.g. PhysicalConsole.ListenToStandardInputAsync on Linux where stdin reads + // don't unblock on process kill). + try + { + Process.StandardInput?.Close(); + } + catch + { + } + try { Process.CancelErrorRead(); diff --git a/test/Microsoft.NET.Sdk.Razor.Tool.Tests/DefaultRequestDispatcherTest.cs b/test/Microsoft.NET.Sdk.Razor.Tool.Tests/DefaultRequestDispatcherTest.cs index 8f3289b9110b..8d222eca8a9a 100644 --- a/test/Microsoft.NET.Sdk.Razor.Tool.Tests/DefaultRequestDispatcherTest.cs +++ b/test/Microsoft.NET.Sdk.Razor.Tool.Tests/DefaultRequestDispatcherTest.cs @@ -358,7 +358,7 @@ public async Task Dispatcher_ProcessSimultaneousConnections_HitsKeepAliveTimeout return connectionTask; } - readySource.SetResult(true); + readySource.TrySetResult(true); return new TaskCompletionSource().Task; }); @@ -382,11 +382,18 @@ public async Task Dispatcher_ProcessSimultaneousConnections_HitsKeepAliveTimeout } }; var keepAlive = TimeSpan.FromSeconds(1); - var dispatcherTask = Task.Run(() => + + // Use Task.Factory.StartNew with LongRunning to run the dispatcher on a dedicated + // OS thread instead of a thread pool thread. The dispatcher's Run() method uses + // blocking Task.WaitAny() which permanently blocks its thread. On Helix CI agents + // running many tests in parallel, blocking a thread pool thread contributes to pool + // starvation, which prevents Task.Delay timer callbacks from firing, causing the + // keep-alive timeout to never complete and the test to hang indefinitely. + var dispatcherTask = Task.Factory.StartNew(() => { var dispatcher = new DefaultRequestDispatcher(connectionHost.Object, compilerHost, CancellationToken.None, eventBus, keepAlive); dispatcher.Run(); - }); + }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default); // Wait for all connections to be created. await readySource.Task; @@ -402,7 +409,20 @@ public async Task Dispatcher_ProcessSimultaneousConnections_HitsKeepAliveTimeout // Act // Now dispatcher should be in an idle state with no active connections. - await dispatcherTask; + // Use a dedicated thread to enforce the timeout, since under extreme thread pool + // starvation on Helix CI, even WaitAsync's timer continuations can't be scheduled. + // A dedicated OS thread with Thread.Join(timeout) uses a kernel wait that works + // regardless of thread pool state. + var completed = false; + var timeoutThread = new Thread(() => + { + completed = dispatcherTask.Wait(TimeSpan.FromSeconds(60)); + }) { IsBackground = true }; + timeoutThread.Start(); + timeoutThread.Join(TimeSpan.FromSeconds(65)); + Assert.True(completed, + "Dispatcher did not shut down within 60 seconds. This likely indicates " + + "thread pool starvation preventing Task.Delay timer callbacks from firing."); // Assert Assert.False(eventBus.HasDetectedBadConnection); diff --git a/test/TestAssets/Directory.Build.targets b/test/TestAssets/Directory.Build.targets index cecd12d3d0c8..2c8eb1f15b8f 100644 --- a/test/TestAssets/Directory.Build.targets +++ b/test/TestAssets/Directory.Build.targets @@ -1,4 +1,21 @@ + + + + + + + diff --git a/test/dotnet-new.IntegrationTests/CommonTemplatesTests.cs b/test/dotnet-new.IntegrationTests/CommonTemplatesTests.cs index 3c21a97d7b7b..51d3001b10a4 100644 --- a/test/dotnet-new.IntegrationTests/CommonTemplatesTests.cs +++ b/test/dotnet-new.IntegrationTests/CommonTemplatesTests.cs @@ -80,6 +80,7 @@ public async Task AllCommonItemsCreate(string expectedTemplateName, string templ .WithCustomScrubbers( ScrubbersDefinition.Empty .AddScrubber(sb => sb.UnixifyNewlines(), "out") + .AddScrubber(sb => sb.ScrubMSBuildDebugLogMessage(), "txt") .AddScrubber((path, content) => { if (path.Replace(Path.DirectorySeparatorChar, '/') == "std-streams/stdout.txt") @@ -224,6 +225,7 @@ public async Task AotVariants(string name, string language) ScrubbersDefinition.Empty .AddScrubber(sb => sb.Replace($"{currentDefaultFramework}", "%FRAMEWORK%")) .AddScrubber(sb => sb.Replace(finalProjectName, "%PROJECT_PATH%").UnixifyDirSeparators().ScrubByRegex("(^ Restored .* \\()(.*)(\\)\\.)", "$1%DURATION%$3", RegexOptions.Multiline), "txt") + .AddScrubber(sb => sb.ScrubMSBuildDebugLogMessage(), "txt") ); VerificationEngine engine = new(_logger); @@ -424,6 +426,7 @@ public async Task FeaturesSupport( .AddScrubber(sb => sb.Replace($"{langVersion}", "%LANG%")) .AddScrubber(sb => sb.Replace($"{framework ?? currentDefaultFramework}", "%FRAMEWORK%")) .AddScrubber(sb => sb.Replace(finalProjectName, "%PROJECT_PATH%").UnixifyDirSeparators().ScrubByRegex("(^ Restored .* \\()(.*)(\\)\\.)", "$1%DURATION%$3", RegexOptions.Multiline), "txt") + .AddScrubber(sb => sb.ScrubMSBuildDebugLogMessage(), "txt") ); VerificationEngine engine = new(_logger); diff --git a/test/dotnet-new.IntegrationTests/DotnetClassTemplateTests.cs b/test/dotnet-new.IntegrationTests/DotnetClassTemplateTests.cs index fd288169f5be..31d4d9bcdd0c 100644 --- a/test/dotnet-new.IntegrationTests/DotnetClassTemplateTests.cs +++ b/test/dotnet-new.IntegrationTests/DotnetClassTemplateTests.cs @@ -77,6 +77,7 @@ public async Task DotnetCSharpClassTemplatesTest( .WithCustomEnvironment(environmentUnderTest!) .WithCustomScrubbers( ScrubbersDefinition.Empty + .AddScrubber(sb => sb.ScrubMSBuildDebugLogMessage(), "txt") .AddScrubber((path, content) => { if (path.Replace(Path.DirectorySeparatorChar, '/') == "std-streams/stdout.txt") @@ -157,6 +158,7 @@ public async Task DotnetVisualBasicClassTemplatesTest( .WithCustomEnvironment(environmentUnderTest!) .WithCustomScrubbers( ScrubbersDefinition.Empty + .AddScrubber(sb => sb.ScrubMSBuildDebugLogMessage(), "txt") .AddScrubber((path, content) => { if (path.Replace(Path.DirectorySeparatorChar, '/') == "std-streams/stdout.txt") diff --git a/test/dotnet-new.IntegrationTests/TemplateEngineSamplesTest.cs b/test/dotnet-new.IntegrationTests/TemplateEngineSamplesTest.cs index 1469a65f90f7..f39fc8c008b7 100644 --- a/test/dotnet-new.IntegrationTests/TemplateEngineSamplesTest.cs +++ b/test/dotnet-new.IntegrationTests/TemplateEngineSamplesTest.cs @@ -63,7 +63,8 @@ public async Task TemplateEngineSamplesProjectTest( .WithCustomEnvironment(environmentUnderTest!) .WithCustomScrubbers( ScrubbersDefinition.Empty - .AddScrubber(sb => sb.Replace(DateTime.Now.ToString("MM/dd/yyyy"), "**/**/****"))); + .AddScrubber(sb => sb.Replace(DateTime.Now.ToString("MM/dd/yyyy"), "**/**/****")) + .AddScrubber(sb => sb.ScrubMSBuildDebugLogMessage(), "txt")); VerificationEngine engine = new(_log); await engine.Execute(options); diff --git a/test/dotnet-new.IntegrationTests/VerifyScrubbers.cs b/test/dotnet-new.IntegrationTests/VerifyScrubbers.cs index b7c51ab40d96..cce088deff16 100644 --- a/test/dotnet-new.IntegrationTests/VerifyScrubbers.cs +++ b/test/dotnet-new.IntegrationTests/VerifyScrubbers.cs @@ -51,6 +51,15 @@ internal static void ScrubByRegex(this StringBuilder output, string pattern, str output.Append(finalOutput); } + /// + /// Removes MSBuild debug log path messages that appear intermittently depending on + /// telemetry/profiling settings, which vary across CI machines causing snapshot mismatches. + /// + internal static void ScrubMSBuildDebugLogMessage(this StringBuilder output) + { + output.ScrubByRegex(@"^\s*MSBuild logs and debug information will be at .*[\r\n]*", "", RegexOptions.Multiline); + } + /// /// Replaces content matching with . /// diff --git a/test/dotnet-watch.Tests/HotReload/ProjectUpdateInProcTests.cs b/test/dotnet-watch.Tests/HotReload/ProjectUpdateInProcTests.cs index f39310095b2b..a322afb75ed3 100644 --- a/test/dotnet-watch.Tests/HotReload/ProjectUpdateInProcTests.cs +++ b/test/dotnet-watch.Tests/HotReload/ProjectUpdateInProcTests.cs @@ -116,10 +116,15 @@ public async Task ProjectAndSourceFileChange_AddProjectReference() AssertEx.ContainsSubstring("Resolving 'Dependency, Version=1.0.0.0'", w.Reporter.ProcessOutput); + // Wait for the fire-and-forget task in CompilationHandler.CompleteApplyOperationAsync + // to finish logging ManagedCodeChangesApplied. The app output arrives before this task + // completes because it's not awaited (line 497 of CompilationHandler.cs). + using var waitCts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await managedCodeChangesApplied.WaitAsync(waitCts.Token); + Assert.Equal(1, projectChangeTriggeredReEvaluation.CurrentCount); Assert.Equal(1, projectsRebuilt.CurrentCount); Assert.Equal(1, projectDependenciesDeployed.CurrentCount); - Assert.Equal(1, managedCodeChangesApplied.CurrentCount); } [Fact] @@ -174,9 +179,14 @@ public async Task ProjectAndSourceFileChange_AddPackageReference() AssertEx.ContainsSubstring("Resolving 'Newtonsoft.Json, Version=13.0.0.0'", w.Reporter.ProcessOutput); + // Wait for the fire-and-forget task in CompilationHandler.CompleteApplyOperationAsync + // to finish logging ManagedCodeChangesApplied. The app output arrives before this task + // completes because it's not awaited (line 497 of CompilationHandler.cs). + using var waitCts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await managedCodeChangesApplied.WaitAsync(waitCts.Token); + Assert.Equal(1, projectChangeTriggeredReEvaluation.CurrentCount); Assert.Equal(0, projectsRebuilt.CurrentCount); Assert.Equal(1, projectDependenciesDeployed.CurrentCount); - Assert.Equal(1, managedCodeChangesApplied.CurrentCount); } }