Fix CI flakiness: MSB4216 task host failures, dotnet-watch hangs, NuGet errors

build/RunTestsOnHelix.cmd

@@ -9,6 +9,10 @@ set DOTNET_ROOT=%HELIX_CORRELATION_PAYLOAD%\d
 set PATH=%DOTNET_ROOT%;%PATH%
 set TestFullMSBuild=%1
 
+REM Set DOTNET_HOST_PATH so MSBuild task hosts can locate the dotnet executable.
+REM Without this, tasks from NuGet packages that use TaskHostFactory fail with MSB4216.
+set DOTNET_HOST_PATH=%DOTNET_ROOT%\dotnet.exe
+
 REM Ensure Visual Studio instances allow preview SDKs
 PowerShell -ExecutionPolicy ByPass -NoProfile -File "%HELIX_CORRELATION_PAYLOAD%\t\eng\enable-preview-sdks.ps1"
 
@@ -35,14 +39,16 @@ dotnet new --debug:ephemeral-hive
 dotnet nuget list source --configfile %TestExecutionDirectory%\nuget.config
 if exist %TestExecutionDirectory%\Testpackages dotnet nuget add source %TestExecutionDirectory%\Testpackages --name testpackages --configfile %TestExecutionDirectory%\nuget.config
 
-dotnet nuget remove source dotnet6-transport --configfile %TestExecutionDirectory%\nuget.config
-dotnet nuget remove source dotnet6-internal-transport --configfile %TestExecutionDirectory%\nuget.config
-dotnet nuget remove source dotnet7-transport --configfile %TestExecutionDirectory%\nuget.config
-dotnet nuget remove source dotnet7-internal-transport --configfile %TestExecutionDirectory%\nuget.config
-dotnet nuget remove source richnav --configfile %TestExecutionDirectory%\nuget.config
-dotnet nuget remove source vs-impl --configfile %TestExecutionDirectory%\nuget.config
-dotnet nuget remove source dotnet-libraries-transport --configfile %TestExecutionDirectory%\nuget.config
-dotnet nuget remove source dotnet-tools-transport --configfile %TestExecutionDirectory%\nuget.config
-dotnet nuget remove source dotnet-libraries --configfile %TestExecutionDirectory%\nuget.config
-dotnet nuget remove source dotnet-eng --configfile %TestExecutionDirectory%\nuget.config
+REM Remove feeds not needed for tests. Errors from non-existent sources
+REM (e.g. internal-transport feeds only present in internal builds) are ignored.
+dotnet nuget remove source dotnet6-transport --configfile %TestExecutionDirectory%\nuget.config 2>nul
+dotnet nuget remove source dotnet6-internal-transport --configfile %TestExecutionDirectory%\nuget.config 2>nul
+dotnet nuget remove source dotnet7-transport --configfile %TestExecutionDirectory%\nuget.config 2>nul
+dotnet nuget remove source dotnet7-internal-transport --configfile %TestExecutionDirectory%\nuget.config 2>nul
+dotnet nuget remove source richnav --configfile %TestExecutionDirectory%\nuget.config 2>nul
+dotnet nuget remove source vs-impl --configfile %TestExecutionDirectory%\nuget.config 2>nul
+dotnet nuget remove source dotnet-libraries-transport --configfile %TestExecutionDirectory%\nuget.config 2>nul
+dotnet nuget remove source dotnet-tools-transport --configfile %TestExecutionDirectory%\nuget.config 2>nul
+dotnet nuget remove source dotnet-libraries --configfile %TestExecutionDirectory%\nuget.config 2>nul
+dotnet nuget remove source dotnet-eng --configfile %TestExecutionDirectory%\nuget.config 2>nul
 dotnet nuget list source --configfile %TestExecutionDirectory%\nuget.config

build/RunTestsOnHelix.sh

@@ -9,6 +9,12 @@ export MicrosoftNETBuildExtensionsTargets=$HELIX_CORRELATION_PAYLOAD/ex/msbuildE
 export DOTNET_ROOT=$HELIX_CORRELATION_PAYLOAD/d
 export PATH=$DOTNET_ROOT:$PATH
 
+# Set DOTNET_HOST_PATH so MSBuild task hosts can locate the dotnet executable.
+# Without this, tasks from NuGet packages that use TaskHostFactory (e.g. ComputeWasmBuildAssets
+# from WebAssembly SDK, ComputeManagedAssemblies from ILLink) fail with MSB4216 on macOS
+# because the task host process cannot find the dotnet host to launch.
+export DOTNET_HOST_PATH=$DOTNET_ROOT/dotnet
+
 export TestExecutionDirectory=$(realpath "$(mktemp -d "${TMPDIR:-/tmp}"/dotnetSdkTests.XXXXXXXX)")
 export DOTNET_CLI_HOME=$TestExecutionDirectory/.dotnet
 cp -a $HELIX_CORRELATION_PAYLOAD/t/TestExecutionDirectoryFiles/. $TestExecutionDirectory/
@@ -22,15 +28,17 @@ dotnet new --debug:ephemeral-hive
 
 dotnet nuget list source --configfile $TestExecutionDirectory/NuGet.config
 dotnet nuget add source $TestExecutionDirectory/Testpackages --configfile $TestExecutionDirectory/NuGet.config
-#Remove feeds not needed for tests
-dotnet nuget remove source dotnet6-transport --configfile $TestExecutionDirectory/NuGet.config
-dotnet nuget remove source dotnet6-internal-transport --configfile $TestExecutionDirectory/NuGet.config
-dotnet nuget remove source dotnet7-transport --configfile $TestExecutionDirectory/NuGet.config
-dotnet nuget remove source dotnet7-internal-transport --configfile $TestExecutionDirectory/NuGet.config
-dotnet nuget remove source richnav --configfile $TestExecutionDirectory/NuGet.config
-dotnet nuget remove source vs-impl --configfile $TestExecutionDirectory/NuGet.config
-dotnet nuget remove source dotnet-libraries-transport --configfile $TestExecutionDirectory/NuGet.config
-dotnet nuget remove source dotnet-tools-transport --configfile $TestExecutionDirectory/NuGet.config
-dotnet nuget remove source dotnet-libraries --configfile $TestExecutionDirectory/NuGet.config
-dotnet nuget remove source dotnet-eng --configfile $TestExecutionDirectory/NuGet.config
+# Remove feeds not needed for tests. Use || true to avoid errors when a source
+# doesn't exist (e.g. internal-transport feeds are only present in internal builds).
+dotnet nuget remove source dotnet6-transport --configfile $TestExecutionDirectory/NuGet.config || true
+dotnet nuget remove source dotnet6-internal-transport --configfile $TestExecutionDirectory/NuGet.config || true
+dotnet nuget remove source dotnet7-transport --configfile $TestExecutionDirectory/NuGet.config || true
+dotnet nuget remove source dotnet7-internal-transport --configfile $TestExecutionDirectory/NuGet.config || true
+dotnet nuget remove source richnav --configfile $TestExecutionDirectory/NuGet.config || true
+dotnet nuget remove source vs-impl --configfile $TestExecutionDirectory/NuGet.config || true
+dotnet nuget remove source dotnet-libraries-transport --configfile $TestExecutionDirectory/NuGet.config || true
+dotnet nuget remove source dotnet-tools-transport --configfile $TestExecutionDirectory/NuGet.config || true
+dotnet nuget remove source dotnet-libraries --configfile $TestExecutionDirectory/NuGet.config || true
+dotnet nuget remove source dotnet-eng --configfile $TestExecutionDirectory/NuGet.config || true
 dotnet nuget list source --configfile $TestExecutionDirectory/NuGet.config
+

src/BlazorWasmSdk/Tasks/GZipCompress.cs

@@ -20,6 +20,10 @@ public class GZipCompress : Task
         [Required]
         public string OutputDirectory { get; set; }
 
+        // Retry count for transient file I/O errors (e.g., antivirus locks on CI machines).
+        private const int MaxRetries = 3;
+        private const int RetryDelayMs = 200;
+
         public override bool Execute()
         {
             CompressedFiles = new ITaskItem[FilesToCompress.Length];
@@ -56,18 +60,31 @@ public override bool Execute()
                     Log.LogMessage(MessageImportance.Low, "Compressing '{0}' because file is newer than '{1}'.", inputFullPath, outputRelativePath);
                 }
 
-                try
+                // Retry on IOException to handle transient file locks from antivirus, file
+                // indexing, or parallel MSBuild nodes on CI machines (see dotnet/sdk#53424).
+                for (int attempt = 1; attempt <= MaxRetries; attempt++)
                 {
-                    using var sourceStream = File.OpenRead(file.ItemSpec);
-                    using var fileStream = File.Create(outputRelativePath);
-                    using var stream = new GZipStream(fileStream, CompressionLevel.Optimal);
+                    try
+                    {
+                        using var sourceStream = File.OpenRead(file.ItemSpec);
+                        using var fileStream = File.Create(outputRelativePath);
+                        using var stream = new GZipStream(fileStream, CompressionLevel.Optimal);
 
-                    sourceStream.CopyTo(stream);
-                }
-                catch (Exception e)
-                {
-                    Log.LogErrorFromException(e);
-                    return;
+                        sourceStream.CopyTo(stream);
+                        return; // Success
+                    }
+                    catch (IOException) when (attempt < MaxRetries)
+                    {
+                        Log.LogMessage(MessageImportance.Low,
+                            "Retrying compression of '{0}' (attempt {1}/{2}) due to transient I/O error.",
+                            file.ItemSpec, attempt, MaxRetries);
+                        Thread.Sleep(RetryDelayMs * attempt);
+                    }
+                    catch (Exception e)
+                    {
+                        Log.LogErrorFromException(e);
+                        return;
+                    }
                 }
             });
 

src/Dotnet.Watch/Watch/UI/PhysicalConsole.cs

@@ -13,7 +13,15 @@ internal sealed class PhysicalConsole : IConsole
     public PhysicalConsole(TestFlags testFlags)
     {
         Console.OutputEncoding = Encoding.UTF8;
-        _ = testFlags.HasFlag(TestFlags.ReadKeyFromStdin) ? ListenToStandardInputAsync() : ListenToConsoleKeyPressAsync();
+
+        if (testFlags.HasFlag(TestFlags.ReadKeyFromStdin))
+        {
+            _ = ListenToStandardInputAsync();
+        }
+        else if (!Console.IsInputRedirected)
+        {
+            _ = ListenToConsoleKeyPressAsync();
+        }
     }
 
     private async Task ListenToStandardInputAsync()

test/Microsoft.DotNet.HotReload.Test.Utilities/AwaitableProcess.cs

@@ -205,6 +205,17 @@ public async ValueTask DisposeAsync()
             Process.ErrorDataReceived -= OnData;
             Process.OutputDataReceived -= OnData;
 
+            // Close stdin before killing the process to unblock any pending stdin reads
+            // (e.g. PhysicalConsole.ListenToStandardInputAsync on Linux where stdin reads
+            // don't unblock on process kill).
+            try
+            {
+                Process.StandardInput?.Close();
+            }
+            catch
+            {
+            }
+
             try
             {
                 Process.CancelErrorRead();

test/Microsoft.NET.Sdk.Razor.Tool.Tests/DefaultRequestDispatcherTest.cs

@@ -358,7 +358,7 @@ public async Task Dispatcher_ProcessSimultaneousConnections_HitsKeepAliveTimeout
                         return connectionTask;
                     }
 
-                    readySource.SetResult(true);
+                    readySource.TrySetResult(true);
                     return new TaskCompletionSource<Connection>().Task;
                 });
 
@@ -382,11 +382,18 @@ public async Task Dispatcher_ProcessSimultaneousConnections_HitsKeepAliveTimeout
                 }
             };
             var keepAlive = TimeSpan.FromSeconds(1);
-            var dispatcherTask = Task.Run(() =>
+
+            // Use Task.Factory.StartNew with LongRunning to run the dispatcher on a dedicated
+            // OS thread instead of a thread pool thread. The dispatcher's Run() method uses
+            // blocking Task.WaitAny() which permanently blocks its thread. On Helix CI agents
+            // running many tests in parallel, blocking a thread pool thread contributes to pool
+            // starvation, which prevents Task.Delay timer callbacks from firing, causing the
+            // keep-alive timeout to never complete and the test to hang indefinitely.
+            var dispatcherTask = Task.Factory.StartNew(() =>
             {
                 var dispatcher = new DefaultRequestDispatcher(connectionHost.Object, compilerHost, CancellationToken.None, eventBus, keepAlive);
                 dispatcher.Run();
-            });
+            }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default);
 
             // Wait for all connections to be created.
             await readySource.Task;
@@ -402,7 +409,20 @@ public async Task Dispatcher_ProcessSimultaneousConnections_HitsKeepAliveTimeout
 
             // Act
             // Now dispatcher should be in an idle state with no active connections.
-            await dispatcherTask;
+            // Use a dedicated thread to enforce the timeout, since under extreme thread pool
+            // starvation on Helix CI, even WaitAsync's timer continuations can't be scheduled.
+            // A dedicated OS thread with Thread.Join(timeout) uses a kernel wait that works
+            // regardless of thread pool state.
+            var completed = false;
+            var timeoutThread = new Thread(() =>
+            {
+                completed = dispatcherTask.Wait(TimeSpan.FromSeconds(60));
+            }) { IsBackground = true };
+            timeoutThread.Start();
+            timeoutThread.Join(TimeSpan.FromSeconds(65));
+            Assert.True(completed,
+                "Dispatcher did not shut down within 60 seconds. This likely indicates " +
+                "thread pool starvation preventing Task.Delay timer callbacks from firing.");
 
             // Assert
             Assert.False(eventBus.HasDetectedBadConnection);

test/TestAssets/Directory.Build.targets

@@ -1,4 +1,21 @@
 <!-- Prevent test asset projects from picking up the repo's root Directory.Build.targets. -->
 <Project>
 
+  <!-- For packable Exe projects (DotNetCliToolReference tools targeting netcoreapp2.2),
+       include the auto-generated runtimeconfig.json in the NuGet package so the dotnet
+       host can find it adjacent to the DLL. This enables RollForward=LatestMajor to
+       work correctly, allowing tools to run on machines that only have .NET 6.0+
+       installed (common on Helix CI agents that lack .NET Core 2.2). Without this,
+       tools fail with FrameworkMissingFailure (exit code 0x80008096) because the host
+       cannot roll forward from 2.2.0 without a runtimeconfig.json specifying the
+       rollForward policy. -->
+  <Target Name="IncludeRuntimeConfigInPackage"
+          AfterTargets="Build"
+          Condition="'$(OutputType)' == 'Exe' AND '$(IsPackable)' == 'true' AND '$(GenerateRuntimeConfigurationFiles)' != 'false'">
+    <ItemGroup>
+      <BuildOutputInPackage Include="$(ProjectRuntimeConfigFilePath)"
+                            Condition="'$(ProjectRuntimeConfigFilePath)' != '' AND Exists('$(ProjectRuntimeConfigFilePath)')" />
+    </ItemGroup>
+  </Target>
+
 </Project>

test/dotnet-new.IntegrationTests/CommonTemplatesTests.cs

@@ -80,6 +80,7 @@ public async Task AllCommonItemsCreate(string expectedTemplateName, string templ
             .WithCustomScrubbers(
                 ScrubbersDefinition.Empty
                     .AddScrubber(sb => sb.UnixifyNewlines(), "out")
+                    .AddScrubber(sb => sb.ScrubMSBuildDebugLogMessage(), "txt")
                     .AddScrubber((path, content) =>
                     {
                         if (path.Replace(Path.DirectorySeparatorChar, '/') == "std-streams/stdout.txt")
@@ -224,6 +225,7 @@ public async Task AotVariants(string name, string language)
                 ScrubbersDefinition.Empty
                     .AddScrubber(sb => sb.Replace($"<TargetFramework>{currentDefaultFramework}</TargetFramework>", "<TargetFramework>%FRAMEWORK%</TargetFramework>"))
                     .AddScrubber(sb => sb.Replace(finalProjectName, "%PROJECT_PATH%").UnixifyDirSeparators().ScrubByRegex("(^  Restored .* \\()(.*)(\\)\\.)", "$1%DURATION%$3", RegexOptions.Multiline), "txt")
+                    .AddScrubber(sb => sb.ScrubMSBuildDebugLogMessage(), "txt")
             );
 
             VerificationEngine engine = new(_logger);
@@ -424,6 +426,7 @@ public async Task FeaturesSupport(
                     .AddScrubber(sb => sb.Replace($"<LangVersion>{langVersion}</LangVersion>", "<LangVersion>%LANG%</LangVersion>"))
                     .AddScrubber(sb => sb.Replace($"<TargetFramework>{framework ?? currentDefaultFramework}</TargetFramework>", "<TargetFramework>%FRAMEWORK%</TargetFramework>"))
                     .AddScrubber(sb => sb.Replace(finalProjectName, "%PROJECT_PATH%").UnixifyDirSeparators().ScrubByRegex("(^  Restored .* \\()(.*)(\\)\\.)", "$1%DURATION%$3", RegexOptions.Multiline), "txt")
+                    .AddScrubber(sb => sb.ScrubMSBuildDebugLogMessage(), "txt")
             );
 
             VerificationEngine engine = new(_logger);

test/dotnet-new.IntegrationTests/DotnetClassTemplateTests.cs

@@ -77,6 +77,7 @@ public async Task DotnetCSharpClassTemplatesTest(
             .WithCustomEnvironment(environmentUnderTest!)
             .WithCustomScrubbers(
                ScrubbersDefinition.Empty
+               .AddScrubber(sb => sb.ScrubMSBuildDebugLogMessage(), "txt")
                .AddScrubber((path, content) =>
                {
                    if (path.Replace(Path.DirectorySeparatorChar, '/') == "std-streams/stdout.txt")
@@ -157,6 +158,7 @@ public async Task DotnetVisualBasicClassTemplatesTest(
             .WithCustomEnvironment(environmentUnderTest!)
             .WithCustomScrubbers(
                ScrubbersDefinition.Empty
+               .AddScrubber(sb => sb.ScrubMSBuildDebugLogMessage(), "txt")
                .AddScrubber((path, content) =>
                {
                    if (path.Replace(Path.DirectorySeparatorChar, '/') == "std-streams/stdout.txt")

test/dotnet-new.IntegrationTests/TemplateEngineSamplesTest.cs

@@ -63,7 +63,8 @@ public async Task TemplateEngineSamplesProjectTest(
             .WithCustomEnvironment(environmentUnderTest!)
             .WithCustomScrubbers(
                ScrubbersDefinition.Empty
-               .AddScrubber(sb => sb.Replace(DateTime.Now.ToString("MM/dd/yyyy"), "**/**/****")));
+               .AddScrubber(sb => sb.Replace(DateTime.Now.ToString("MM/dd/yyyy"), "**/**/****"))
+               .AddScrubber(sb => sb.ScrubMSBuildDebugLogMessage(), "txt"));
 
             VerificationEngine engine = new(_log);
             await engine.Execute(options);

test/dotnet-new.IntegrationTests/VerifyScrubbers.cs

@@ -51,6 +51,15 @@ internal static void ScrubByRegex(this StringBuilder output, string pattern, str
             output.Append(finalOutput);
         }
 
+        /// <summary>
+        /// Removes MSBuild debug log path messages that appear intermittently depending on
+        /// telemetry/profiling settings, which vary across CI machines causing snapshot mismatches.
+        /// </summary>
+        internal static void ScrubMSBuildDebugLogMessage(this StringBuilder output)
+        {
+            output.ScrubByRegex(@"^\s*MSBuild logs and debug information will be at .*[\r\n]*", "", RegexOptions.Multiline);
+        }
+
         /// <summary>
         /// Replaces content matching <paramref name="textToReplace"/> with <paramref name="replacement"/>.
         /// </summary>

test/dotnet-watch.Tests/HotReload/ProjectUpdateInProcTests.cs

@@ -116,10 +116,15 @@ public async Task ProjectAndSourceFileChange_AddProjectReference()
 
         AssertEx.ContainsSubstring("Resolving 'Dependency, Version=1.0.0.0'", w.Reporter.ProcessOutput);
 
+        // Wait for the fire-and-forget task in CompilationHandler.CompleteApplyOperationAsync
+        // to finish logging ManagedCodeChangesApplied. The app output arrives before this task
+        // completes because it's not awaited (line 497 of CompilationHandler.cs).
+        using var waitCts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+        await managedCodeChangesApplied.WaitAsync(waitCts.Token);
+
         Assert.Equal(1, projectChangeTriggeredReEvaluation.CurrentCount);
         Assert.Equal(1, projectsRebuilt.CurrentCount);
         Assert.Equal(1, projectDependenciesDeployed.CurrentCount);
-        Assert.Equal(1, managedCodeChangesApplied.CurrentCount);
     }
 
     [Fact]
@@ -174,9 +179,14 @@ public async Task ProjectAndSourceFileChange_AddPackageReference()
 
         AssertEx.ContainsSubstring("Resolving 'Newtonsoft.Json, Version=13.0.0.0'", w.Reporter.ProcessOutput);
 
+        // Wait for the fire-and-forget task in CompilationHandler.CompleteApplyOperationAsync
+        // to finish logging ManagedCodeChangesApplied. The app output arrives before this task
+        // completes because it's not awaited (line 497 of CompilationHandler.cs).
+        using var waitCts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+        await managedCodeChangesApplied.WaitAsync(waitCts.Token);
+
         Assert.Equal(1, projectChangeTriggeredReEvaluation.CurrentCount);
         Assert.Equal(0, projectsRebuilt.CurrentCount);
         Assert.Equal(1, projectDependenciesDeployed.CurrentCount);
-        Assert.Equal(1, managedCodeChangesApplied.CurrentCount);
     }
 }