Skip to content

Commit cd06388

Browse files
timcasselltimcassell
authored
Add InProcessSettings option to run on same thread. (#2938)
Remove timeout. Auto-select toolchain based on IsAot.
1 parent ce48f3e commit cd06388

27 files changed

+143
-218
lines changed

‎samples/BenchmarkDotNet.Samples/IntroInProcess.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ public Config()
2222

2323
AddJob(Job.MediumRun
2424
.WithLaunchCount(1)
25-
.WithToolchain(InProcessEmitToolchain.Instance)
25+
.WithToolchain(InProcessEmitToolchain.Default)
2626
.WithId("InProcess"));
2727
}
2828
}

‎samples/BenchmarkDotNet.Samples/IntroInProcessWrongEnv.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ public Config()
2626
AddJob(Job.MediumRun
2727
.WithLaunchCount(1)
2828
.WithPlatform(wrongPlatform)
29-
.WithToolchain(InProcessEmitToolchain.Instance)
29+
.WithToolchain(InProcessEmitToolchain.Default)
3030
.WithId("InProcess"));
3131

3232
AddValidator(InProcessValidator.DontFailOnError);

‎src/BenchmarkDotNet/Attributes/Jobs/InProcessAttribute.cs‎

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,34 @@
11
using System;
22
using BenchmarkDotNet.Jobs;
3+
using BenchmarkDotNet.Portability;
4+
using BenchmarkDotNet.Toolchains.InProcess.Emit;
5+
using BenchmarkDotNet.Toolchains.InProcess.NoEmit;
36

4-
namespace BenchmarkDotNet.Attributes
7+
namespace BenchmarkDotNet.Attributes;
8+
9+
public enum InProcessToolchainType
10+
{
11+
Auto,
12+
Emit,
13+
NoEmit,
14+
}
15+
16+
[AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly)]
17+
public class InProcessAttribute(
18+
InProcessToolchainType toolchainType = InProcessToolchainType.Auto,
19+
bool executeOnSeparateThread = true)
20+
: JobConfigBaseAttribute(GetJob(toolchainType, executeOnSeparateThread))
521
{
6-
[AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly)]
7-
public class InProcessAttribute : JobConfigBaseAttribute
22+
internal static Job GetJob(InProcessToolchainType toolchainType, bool executeOnSeparateThread)
823
{
9-
public InProcessAttribute(bool dontLogOutput = false) : base(dontLogOutput ? Job.InProcessDontLogOutput : Job.InProcess)
24+
if (toolchainType == InProcessToolchainType.Auto)
1025
{
26+
toolchainType = RuntimeInformation.IsAot
27+
? InProcessToolchainType.NoEmit
28+
: InProcessToolchainType.Emit;
1129
}
30+
return toolchainType == InProcessToolchainType.Emit
31+
? Job.Default.WithToolchain(new InProcessEmitToolchain(new() { ExecuteOnSeparateThread = executeOnSeparateThread }))
32+
: Job.Default.WithToolchain(new InProcessNoEmitToolchain(new() { ExecuteOnSeparateThread = executeOnSeparateThread }));
1233
}
1334
}

‎src/BenchmarkDotNet/Configs/DebugConfig.cs‎

Lines changed: 21 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -25,15 +25,11 @@ namespace BenchmarkDotNet.Configs
2525
[PublicAPI]
2626
public class DebugInProcessConfig : DebugConfig
2727
{
28-
public override IEnumerable<Job> GetJobs()
29-
=> new[]
30-
{
31-
Job.Default
32-
.WithToolchain(
33-
new InProcessEmitToolchain(
34-
TimeSpan.FromHours(1), // 1h should be enough to debug the benchmark
35-
true))
36-
};
28+
public override IEnumerable<Job> GetJobs() =>
29+
[
30+
Job.Default
31+
.WithToolchain(InProcessEmitToolchain.Default)
32+
];
3733
}
3834

3935
/// <summary>
@@ -42,29 +38,28 @@ public override IEnumerable<Job> GetJobs()
4238
[PublicAPI]
4339
public class DebugBuildConfig : DebugConfig
4440
{
45-
public override IEnumerable<Job> GetJobs()
46-
=> new[]
47-
{
48-
Job.Default
49-
.WithCustomBuildConfiguration("Debug") // will do `-c Debug everywhere`
50-
};
41+
public override IEnumerable<Job> GetJobs() =>
42+
[
43+
Job.Default
44+
.WithCustomBuildConfiguration("Debug") // will do `-c Debug everywhere`
45+
];
5146
}
5247

5348
public abstract class DebugConfig : IConfig
5449
{
55-
private readonly static Conclusion[] emptyConclusion = Array.Empty<Conclusion>();
50+
private readonly static Conclusion[] emptyConclusion = [];
5651
public abstract IEnumerable<Job> GetJobs();
5752

58-
public IEnumerable<IValidator> GetValidators() => Array.Empty<IValidator>();
53+
public IEnumerable<IValidator> GetValidators() => [];
5954
public IEnumerable<IColumnProvider> GetColumnProviders() => DefaultColumnProviders.Instance;
60-
public IEnumerable<IExporter> GetExporters() => Array.Empty<IExporter>();
61-
public IEnumerable<ILogger> GetLoggers() => new[] { ConsoleLogger.Default };
62-
public IEnumerable<IDiagnoser> GetDiagnosers() => Array.Empty<IDiagnoser>();
63-
public IEnumerable<IAnalyser> GetAnalysers() => Array.Empty<IAnalyser>();
64-
public IEnumerable<HardwareCounter> GetHardwareCounters() => Array.Empty<HardwareCounter>();
65-
public IEnumerable<EventProcessor> GetEventProcessors() => Array.Empty<EventProcessor>();
66-
public IEnumerable<IFilter> GetFilters() => Array.Empty<IFilter>();
67-
public IEnumerable<IColumnHidingRule> GetColumnHidingRules() => Array.Empty<IColumnHidingRule>();
55+
public IEnumerable<IExporter> GetExporters() => [];
56+
public IEnumerable<ILogger> GetLoggers() => [ConsoleLogger.Default];
57+
public IEnumerable<IDiagnoser> GetDiagnosers() => [];
58+
public IEnumerable<IAnalyser> GetAnalysers() => [];
59+
public IEnumerable<HardwareCounter> GetHardwareCounters() => [];
60+
public IEnumerable<EventProcessor> GetEventProcessors() => [];
61+
public IEnumerable<IFilter> GetFilters() => [];
62+
public IEnumerable<IColumnHidingRule> GetColumnHidingRules() => [];
6863

6964
public IOrderer Orderer => DefaultOrderer.Instance;
7065
public ICategoryDiscoverer? CategoryDiscoverer => DefaultCategoryDiscoverer.Instance;
@@ -76,7 +71,7 @@ public abstract class DebugConfig : IConfig
7671
public string ArtifactsPath => null; // DefaultConfig.ArtifactsPath will be used if the user does not specify it in explicit way
7772

7873
public CultureInfo CultureInfo => null;
79-
public IEnumerable<BenchmarkLogicalGroupRule> GetLogicalGroupRules() => Array.Empty<BenchmarkLogicalGroupRule>();
74+
public IEnumerable<BenchmarkLogicalGroupRule> GetLogicalGroupRules() => [];
8075

8176
public ConfigOptions Options => ConfigOptions.KeepBenchmarkFiles | ConfigOptions.DisableOptimizationsValidator;
8277

‎src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -464,7 +464,7 @@ private static IEnumerable<Job> Expand(Job baseJob, CommandLineOptions options,
464464
{
465465
if (options.RunInProcess)
466466
{
467-
yield return baseJob.WithToolchain(InProcessEmitToolchain.Instance);
467+
yield return Attributes.InProcessAttribute.GetJob(Attributes.InProcessToolchainType.Auto, true);
468468
}
469469
else if (options.ClrVersion.IsNotBlank())
470470
{

‎src/BenchmarkDotNet/Jobs/InfrastructureMode.cs‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
using System;
2-
using System.Collections.Generic;
3-
using System.ComponentModel;
1+
using System.Collections.Generic;
42
using System.Diagnostics.CodeAnalysis;
53
using BenchmarkDotNet.Characteristics;
64
using BenchmarkDotNet.Engines;
@@ -23,8 +21,7 @@ public sealed class InfrastructureMode : JobMode<InfrastructureMode>
2321
public static readonly Characteristic<string> BuildConfigurationCharacteristic = CreateCharacteristic<string>(nameof(BuildConfiguration));
2422
public static readonly Characteristic<IReadOnlyList<Argument>> ArgumentsCharacteristic = CreateCharacteristic<IReadOnlyList<Argument>>(nameof(Arguments));
2523

26-
public static readonly InfrastructureMode InProcess = new InfrastructureMode(InProcessEmitToolchain.Instance);
27-
public static readonly InfrastructureMode InProcessDontLogOutput = new InfrastructureMode(InProcessEmitToolchain.DontLogOutput);
24+
public static readonly InfrastructureMode InProcess = new InfrastructureMode(InProcessEmitToolchain.Default);
2825

2926
public InfrastructureMode() { }
3027

‎src/BenchmarkDotNet/Jobs/Job.cs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ public sealed class Job : JobMode<Job>
2828

2929
// Infrastructure
3030
public static readonly Job InProcess = new Job(nameof(InProcess), InfrastructureMode.InProcess);
31-
public static readonly Job InProcessDontLogOutput = new Job(nameof(InProcessDontLogOutput), InfrastructureMode.InProcessDontLogOutput);
3231

3332
public Job() : this("") { }
3433

‎src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitExecutor.cs‎

Lines changed: 17 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,7 @@
44
using System.Reflection;
55
using System.Threading;
66
using BenchmarkDotNet.Detectors;
7-
using BenchmarkDotNet.Diagnosers;
87
using BenchmarkDotNet.Engines;
9-
using BenchmarkDotNet.Environments;
108
using BenchmarkDotNet.Extensions;
119
using BenchmarkDotNet.Jobs;
1210
using BenchmarkDotNet.Loggers;
@@ -16,80 +14,37 @@
1614

1715
namespace BenchmarkDotNet.Toolchains.InProcess.Emit
1816
{
19-
/// <summary>
20-
/// Implementation of <see cref="IExecutor" /> for in-process benchmarks.
21-
/// </summary>
22-
public class InProcessEmitExecutor : IExecutor
17+
internal class InProcessEmitExecutor(bool executeOnSeparateThread) : IExecutor
2318
{
24-
private static readonly TimeSpan UnderDebuggerTimeout = TimeSpan.FromDays(1);
25-
private static readonly TimeSpan UnderProfilerTimeout = TimeSpan.FromDays(1);
26-
27-
/// <summary> Default timeout for in-process benchmarks. </summary>
28-
public static readonly TimeSpan DefaultTimeout = TimeSpan.FromMinutes(5);
29-
30-
/// <summary>Initializes a new instance of the <see cref="InProcessEmitExecutor" /> class.</summary>
31-
/// <param name="timeout">Timeout for the run.</param>
32-
/// <param name="logOutput"><c>true</c> if the output should be logged.</param>
33-
public InProcessEmitExecutor(TimeSpan timeout, bool logOutput)
34-
{
35-
if (timeout == TimeSpan.Zero)
36-
timeout = DefaultTimeout;
37-
38-
ExecutionTimeout = timeout;
39-
LogOutput = logOutput;
40-
}
41-
42-
/// <summary>Timeout for the run.</summary>
43-
/// <value>The timeout for the run.</value>
44-
public TimeSpan ExecutionTimeout { get; }
45-
46-
/// <summary>Gets a value indicating whether the output should be logged.</summary>
47-
/// <value><c>true</c> if the output should be logged; otherwise, <c>false</c>.</value>
48-
public bool LogOutput { get; }
49-
50-
/// <summary>Executes the specified benchmark.</summary>
5119
public ExecuteResult Execute(ExecuteParameters executeParameters)
5220
{
53-
// TODO: preallocate buffer for output (no direct logging)?
54-
var hostLogger = LogOutput ? executeParameters.Logger : NullLogger.Instance;
55-
var host = new InProcessHost(executeParameters.BenchmarkCase, hostLogger, executeParameters.Diagnoser);
21+
var host = new InProcessHost(executeParameters.BenchmarkCase, executeParameters.Logger, executeParameters.Diagnoser);
5622

5723
int exitCode = -1;
58-
var runThread = new Thread(() => exitCode = ExecuteCore(host, executeParameters));
59-
60-
if (executeParameters.BenchmarkCase.Descriptor.WorkloadMethod
61-
.GetCustomAttributes<STAThreadAttribute>(false)
62-
.Any() &&
63-
OsDetector.IsWindows())
24+
if (executeOnSeparateThread)
6425
{
65-
runThread.SetApartmentState(ApartmentState.STA);
66-
}
26+
var runThread = new Thread(() => exitCode = ExecuteCore(host, executeParameters));
6727

68-
runThread.IsBackground = true;
69-
70-
var timeout = GetTimeout(executeParameters);
71-
72-
runThread.Start();
28+
if (executeParameters.BenchmarkCase.Descriptor.WorkloadMethod.GetCustomAttributes<STAThreadAttribute>(false).Any()
29+
&& OsDetector.IsWindows())
30+
{
31+
runThread.SetApartmentState(ApartmentState.STA);
32+
}
7333

74-
if (!runThread.Join((int)timeout.TotalMilliseconds))
75-
throw new InvalidOperationException(
76-
$"Benchmark {executeParameters.BenchmarkCase.DisplayInfo} takes too long to run. " +
77-
"Prefer to use out-of-process toolchains for long-running benchmarks.");
34+
runThread.IsBackground = true;
7835

36+
runThread.Start();
37+
runThread.Join();
38+
}
39+
else
40+
{
41+
exitCode = ExecuteCore(host, executeParameters);
42+
}
7943
host.HandleInProcessDiagnoserResults(executeParameters.BenchmarkCase, executeParameters.CompositeInProcessDiagnoser);
8044

8145
return ExecuteResult.FromRunResults(host.RunResults, exitCode);
8246
}
8347

84-
private TimeSpan GetTimeout(ExecuteParameters executeParameters)
85-
{
86-
if (HostEnvironmentInfo.GetCurrent().HasAttachedDebugger)
87-
return UnderDebuggerTimeout;
88-
if (executeParameters.Diagnoser is IProfiler)
89-
return UnderProfilerTimeout;
90-
return ExecutionTimeout;
91-
}
92-
9348
private int ExecuteCore(IHost host, ExecuteParameters parameters)
9449
{
9550
int exitCode = -1;

‎src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitSettings.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
namespace BenchmarkDotNet.Toolchains.InProcess.Emit;
2+
3+
public class InProcessEmitSettings : InProcessSettings
4+
{
5+
}

‎src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitToolchain.cs‎

Lines changed: 12 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,23 @@
1-
using System;
2-
using JetBrains.Annotations;
1+
using JetBrains.Annotations;
32

43
namespace BenchmarkDotNet.Toolchains.InProcess.Emit
54
{
5+
/// <summary>
6+
/// An <see cref="IToolchain"/> to run the benchmarks in-process by emitting IL.
7+
/// </summary>
68
[PublicAPI]
79
public class InProcessEmitToolchain : Toolchain
810
{
9-
/// <summary>The default toolchain instance.</summary>
10-
public static readonly IToolchain Instance = new InProcessEmitToolchain(true);
11-
12-
/// <summary>The toolchain instance without output logging.</summary>
13-
public static readonly IToolchain DontLogOutput = new InProcessEmitToolchain(false);
14-
15-
/// <summary>Initializes a new instance of the <see cref="InProcessEmitToolchain" /> class.</summary>
16-
/// <param name="logOutput"><c>true</c> if the output should be logged.</param>
17-
public InProcessEmitToolchain(bool logOutput) :
18-
this(TimeSpan.Zero, logOutput)
19-
{ }
11+
/// <summary>A toolchain instance with default settings.</summary>
12+
public static readonly IToolchain Default = new InProcessEmitToolchain(new() { ExecuteOnSeparateThread = true });
2013

2114
/// <summary>Initializes a new instance of the <see cref="InProcessEmitToolchain" /> class.</summary>
22-
/// <param name="timeout">Timeout for the run.</param>
23-
/// <param name="logOutput"><c>true</c> if the output should be logged.</param>
24-
public InProcessEmitToolchain(TimeSpan timeout, bool logOutput) :
25-
base(
26-
nameof(InProcessEmitToolchain),
27-
new InProcessEmitGenerator(),
28-
new InProcessEmitBuilder(),
29-
new InProcessEmitExecutor(timeout, logOutput))
15+
/// <param name="settings">The settings to use for the toolchain.</param>
16+
public InProcessEmitToolchain(InProcessEmitSettings settings) : base(
17+
nameof(InProcessEmitToolchain),
18+
new InProcessEmitGenerator(),
19+
new InProcessEmitBuilder(),
20+
new InProcessEmitExecutor(settings.ExecuteOnSeparateThread))
3021
{
3122
}
3223

0 commit comments

Comments
 (0)