Improve BuildTimeout (#1906)

* don't try to pass build timeout to every builder, move it to IConfig so it can be always resolved at run time

* implement timeout support for Roslyn toolchain

* remove redundant WasmBuilder

Author: adamsitnik

src/BenchmarkDotNet/Configs/DebugConfig.cs

@@ -66,6 +66,7 @@ public abstract class DebugConfig : IConfig
         public IOrderer Orderer => DefaultOrderer.Instance;
         public SummaryStyle SummaryStyle => SummaryStyle.Default;
         public ConfigUnionRule UnionRule => ConfigUnionRule.Union;
+        public TimeSpan BuildTimeout => DefaultConfig.Instance.BuildTimeout;
 
         public string ArtifactsPath
         {

src/BenchmarkDotNet/Configs/DefaultConfig.cs

@@ -78,6 +78,8 @@ public IEnumerable<IValidator> GetValidators()
 
         public SummaryStyle SummaryStyle => SummaryStyle.Default;
 
+        public TimeSpan BuildTimeout => TimeSpan.FromSeconds(120);
+
         public string ArtifactsPath
         {
             get

src/BenchmarkDotNet/Configs/IConfig.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Globalization;
 using BenchmarkDotNet.Analysers;
 using BenchmarkDotNet.Columns;
@@ -45,5 +46,10 @@ public interface IConfig
         /// a set of custom flags that can enable/disable various settings
         /// </summary>
         ConfigOptions Options { get; }
+
+        /// <summary>
+        /// the auto-generated project build timeout
+        /// </summary>
+        TimeSpan BuildTimeout { get; }
     }
 }

src/BenchmarkDotNet/Configs/ImmutableConfig.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Globalization;
 using System.Linq;
@@ -48,7 +49,8 @@ internal ImmutableConfig(
             CultureInfo cultureInfo,
             IOrderer orderer,
             SummaryStyle summaryStyle,
-            ConfigOptions options)
+            ConfigOptions options,
+            TimeSpan buildTimeout)
         {
             columnProviders = uniqueColumnProviders;
             loggers = uniqueLoggers;
@@ -66,6 +68,7 @@ internal ImmutableConfig(
             Orderer = orderer;
             SummaryStyle = summaryStyle;
             Options = options;
+            BuildTimeout = buildTimeout;
         }
 
         public ConfigUnionRule UnionRule { get; }
@@ -74,6 +77,7 @@ internal ImmutableConfig(
         public ConfigOptions Options { get; }
         [NotNull] public IOrderer Orderer { get; }
         public SummaryStyle SummaryStyle { get; }
+        public TimeSpan BuildTimeout { get; }
 
         public IEnumerable<IColumnProvider> GetColumnProviders() => columnProviders;
         public IEnumerable<IExporter> GetExporters() => exporters;

src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs

@@ -65,7 +65,8 @@ public static ImmutableConfig Create(IConfig source)
                 source.CultureInfo,
                 source.Orderer ?? DefaultOrderer.Instance,
                 source.SummaryStyle ?? SummaryStyle.Default,
-                source.Options
+                source.Options,
+                source.BuildTimeout
             );
         }
 

src/BenchmarkDotNet/Configs/ManualConfig.cs

@@ -48,6 +48,7 @@ public class ManualConfig : IConfig
         [PublicAPI] public CultureInfo CultureInfo { get; set; }
         [PublicAPI] public IOrderer Orderer { get; set; }
         [PublicAPI] public SummaryStyle SummaryStyle { get; set; }
+        [PublicAPI] public TimeSpan BuildTimeout { get; set; } = DefaultConfig.Instance.BuildTimeout;
 
         public ManualConfig WithOption(ConfigOptions option, bool value)
         {
@@ -85,6 +86,12 @@ public ManualConfig WithOrderer(IOrderer orderer)
             return this;
         }
 
+        public ManualConfig WithBuildTimeout(TimeSpan buildTimeout)
+        {
+            BuildTimeout = buildTimeout;
+            return this;
+        }
+
         [EditorBrowsable(EditorBrowsableState.Never)]
         [Obsolete("This method will soon be removed, please start using .AddColumn() instead.")]
         public void Add(params IColumn[] newColumns) => AddColumn(newColumns);
@@ -218,6 +225,7 @@ public void Add(IConfig config)
             SummaryStyle = config.SummaryStyle ?? SummaryStyle;
             logicalGroupRules.AddRange(config.GetLogicalGroupRules());
             Options |= config.Options;
+            BuildTimeout = GetBuildTimeout(BuildTimeout, config.BuildTimeout);
         }
 
         /// <summary>
@@ -263,5 +271,10 @@ public static ManualConfig Union(IConfig globalConfig, IConfig localConfig)
             }
             return manualConfig;
         }
+
+        private static TimeSpan GetBuildTimeout(TimeSpan current, TimeSpan other)
+            => current == DefaultConfig.Instance.BuildTimeout
+                ? other
+                : TimeSpan.FromMilliseconds(Math.Max(current.TotalMilliseconds, other.TotalMilliseconds));
     }
 }

src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs

@@ -245,6 +245,9 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig globalCo
             if (options.MaxParameterColumnWidth.HasValue)
                 config.WithSummaryStyle(SummaryStyle.Default.WithMaxParameterColumnWidth(options.MaxParameterColumnWidth.Value));
 
+            if (options.TimeOutInSeconds.HasValue)
+                config.WithBuildTimeout(TimeSpan.FromSeconds(options.TimeOutInSeconds.Value));
+
             return config;
         }
 
@@ -324,8 +327,6 @@ private static IEnumerable<Job> Expand(Job baseJob, CommandLineOptions options)
 
         private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, CommandLineOptions options)
         {
-            TimeSpan? timeOut = options.TimeOutInSeconds.HasValue ? TimeSpan.FromSeconds(options.TimeOutInSeconds.Value) : default(TimeSpan?);
-
             if (!TryParse(runtimeId, out RuntimeMoniker runtimeMoniker))
             {
                 throw new InvalidOperationException("Impossible, already validated by the Validate method");
@@ -341,7 +342,7 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma
                 case RuntimeMoniker.Net48:
                     return baseJob
                         .WithRuntime(runtimeMoniker.GetRuntime())
-                        .WithToolchain(CsProjClassicNetToolchain.From(runtimeId, options.RestorePath?.FullName, timeOut));
+                        .WithToolchain(CsProjClassicNetToolchain.From(runtimeId, options.RestorePath?.FullName));
                 case RuntimeMoniker.NetCoreApp20:
                 case RuntimeMoniker.NetCoreApp21:
                 case RuntimeMoniker.NetCoreApp22:
@@ -355,7 +356,7 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma
                 case RuntimeMoniker.Net70:
                     return baseJob
                         .WithRuntime(runtimeMoniker.GetRuntime())
-                        .WithToolchain(CsProjCoreToolchain.From(new NetCoreAppSettings(runtimeId, null, runtimeId, options.CliPath?.FullName, options.RestorePath?.FullName, timeOut)));
+                        .WithToolchain(CsProjCoreToolchain.From(new NetCoreAppSettings(runtimeId, null, runtimeId, options.CliPath?.FullName, options.RestorePath?.FullName)));
                 case RuntimeMoniker.Mono:
                     return baseJob.WithRuntime(new MonoRuntime("Mono", options.MonoPath?.FullName));
                 case RuntimeMoniker.CoreRt20:
@@ -380,33 +381,30 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma
                     else
                         builder.UseCoreRtNuGet();
 
-                    if (timeOut.HasValue)
-                        builder.Timeout(timeOut.Value);
-
                     var runtime = runtimeMoniker.GetRuntime();
                     builder.TargetFrameworkMoniker(runtime.MsBuildMoniker);
 
                     return baseJob.WithRuntime(runtime).WithToolchain(builder.ToToolchain());
                 case RuntimeMoniker.Wasm:
-                    return MakeWasmJob(baseJob, options, timeOut, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net5.0");
+                    return MakeWasmJob(baseJob, options, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net5.0");
                 case RuntimeMoniker.WasmNet50:
-                    return MakeWasmJob(baseJob, options, timeOut, "net5.0");
+                    return MakeWasmJob(baseJob, options, "net5.0");
                 case RuntimeMoniker.WasmNet60:
-                    return MakeWasmJob(baseJob, options, timeOut, "net6.0");
+                    return MakeWasmJob(baseJob, options, "net6.0");
                 case RuntimeMoniker.WasmNet70:
-                    return MakeWasmJob(baseJob, options, timeOut, "net7.0");
+                    return MakeWasmJob(baseJob, options, "net7.0");
                 case RuntimeMoniker.MonoAOTLLVM:
-                    return MakeMonoAOTLLVMJob(baseJob, options, timeOut, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net6.0");
+                    return MakeMonoAOTLLVMJob(baseJob, options, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net6.0");
                 case RuntimeMoniker.MonoAOTLLVMNet60:
-                    return MakeMonoAOTLLVMJob(baseJob, options, timeOut, "net6.0");
+                    return MakeMonoAOTLLVMJob(baseJob, options, "net6.0");
                 case RuntimeMoniker.MonoAOTLLVMNet70:
-                    return MakeMonoAOTLLVMJob(baseJob, options, timeOut, "net7.0");
+                    return MakeMonoAOTLLVMJob(baseJob, options, "net7.0");
                 default:
                     throw new NotSupportedException($"Runtime {runtimeId} is not supported");
             }
         }
 
-        private static Job MakeMonoAOTLLVMJob(Job baseJob, CommandLineOptions options, TimeSpan? timeOut, string msBuildMoniker)
+        private static Job MakeMonoAOTLLVMJob(Job baseJob, CommandLineOptions options, string msBuildMoniker)
         {
             var monoAotLLVMRuntime = new MonoAotLLVMRuntime(aotCompilerPath: options.AOTCompilerPath, msBuildMoniker: msBuildMoniker);
 
@@ -417,15 +415,14 @@ private static Job MakeMonoAOTLLVMJob(Job baseJob, CommandLineOptions options, T
                 name: monoAotLLVMRuntime.Name,
                 customDotNetCliPath: options.CliPath?.FullName,
                 packagesPath: options.RestorePath?.FullName,
-                timeout: timeOut ?? NetCoreAppSettings.DefaultBuildTimeout,
                 customRuntimePack: options.CustomRuntimePack,
                 aotCompilerPath: options.AOTCompilerPath.ToString(),
                 aotCompilerMode: options.AOTCompilerMode));
 
             return baseJob.WithRuntime(monoAotLLVMRuntime).WithToolchain(toolChain);
         }
 
-        private static Job MakeWasmJob(Job baseJob, CommandLineOptions options, TimeSpan? timeOut, string msBuildMoniker)
+        private static Job MakeWasmJob(Job baseJob, CommandLineOptions options, string msBuildMoniker)
         {
             bool wasmAot = options.AOTCompilerMode == MonoAotCompilerMode.wasm;
 
@@ -442,7 +439,6 @@ private static Job MakeWasmJob(Job baseJob, CommandLineOptions options, TimeSpan
                 name: wasmRuntime.Name,
                 customDotNetCliPath: options.CliPath?.FullName,
                 packagesPath: options.RestorePath?.FullName,
-                timeout: timeOut ?? NetCoreAppSettings.DefaultBuildTimeout,
                 customRuntimePack: options.CustomRuntimePack,
                 aotCompilerMode: options.AOTCompilerMode));
 

src/BenchmarkDotNet/Running/BuildPartition.cs

@@ -56,11 +56,15 @@ public BuildPartition(BenchmarkBuildInfo[] benchmarks, IResolver resolver)
             || (RepresentativeBenchmarkCase.Job.Infrastructure.TryGetToolchain(out var toolchain) && (toolchain is RoslynToolchain || toolchain is CsProjClassicNetToolchain));
 
         public Runtime Runtime => RepresentativeBenchmarkCase.Job.Environment.HasValue(EnvironmentMode.RuntimeCharacteristic)
-                ? RepresentativeBenchmarkCase.Job.Environment.Runtime
-                : RuntimeInformation.GetCurrentRuntime();
+            ? RepresentativeBenchmarkCase.Job.Environment.Runtime
+            : RuntimeInformation.GetCurrentRuntime();
 
         public bool IsCustomBuildConfiguration => BuildConfiguration != InfrastructureMode.ReleaseConfigurationName;
 
+        public TimeSpan Timeout => IsCoreRT && RepresentativeBenchmarkCase.Config.BuildTimeout == DefaultConfig.Instance.BuildTimeout
+            ? TimeSpan.FromMinutes(5) // downloading all CoreRT dependencies can take a LOT of time
+            : RepresentativeBenchmarkCase.Config.BuildTimeout;
+
         public override string ToString() => RepresentativeBenchmarkCase.Job.DisplayInfo;
 
         private static string GetResolvedAssemblyLocation(Assembly assembly) =>

src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchain.cs

@@ -45,13 +45,12 @@ internal CoreRtToolchain(string displayName,
             string runtimeFrameworkVersion, string targetFrameworkMoniker, string runtimeIdentifier,
             string customDotNetCliPath, string packagesRestorePath,
             Dictionary<string, string> feeds, bool useNuGetClearTag, bool useTempFolderForRestore,
-            TimeSpan timeout,
             bool rootAllApplicationAssemblies, bool ilcGenerateCompleteTypeMetadata, bool ilcGenerateStackTraceData)
             : base(displayName,
                 new Generator(coreRtVersion, useCppCodeGenerator, runtimeFrameworkVersion, targetFrameworkMoniker, customDotNetCliPath,
                     runtimeIdentifier, feeds, useNuGetClearTag, useTempFolderForRestore, packagesRestorePath,
                     rootAllApplicationAssemblies, ilcGenerateCompleteTypeMetadata, ilcGenerateStackTraceData),
-                new DotNetCliPublisher(customDotNetCliPath, GetExtraArguments(useCppCodeGenerator, runtimeIdentifier), GetEnvironmentVariables(ilcPath), timeout),
+                new DotNetCliPublisher(customDotNetCliPath, GetExtraArguments(useCppCodeGenerator, runtimeIdentifier), GetEnvironmentVariables(ilcPath)),
                 new Executor())
         {
             IlcPath = ilcPath;

src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchainBuilder.cs

@@ -144,7 +144,6 @@ public override IToolchain ToToolchain()
                 feeds: Feeds,
                 useNuGetClearTag: useNuGetClearTag,
                 useTempFolderForRestore: useTempFolderForRestore,
-                timeout: timeout ?? TimeSpan.FromMinutes(5),  // downloading all CoreRT dependencies can take a LOT of time
                 rootAllApplicationAssemblies: rootAllApplicationAssemblies,
                 ilcGenerateCompleteTypeMetadata: ilcGenerateCompleteTypeMetadata,
                 ilcGenerateStackTraceData: ilcGenerateStackTraceData