Skip to content

Commit c9dec4d

Browse files
timcasselltimcassell
authored
Extra iteration diagnosers (#2934)
* Added `BeforeExtraIteration` and `AfterExtraIteration` signals. Refactored `ThreadingDiagnoser` and `ExceptionDiagnoser` to implement `IInProcessDiagnoser` using the new signals. * Fixed tests. * Reduce number of tests in default toolchain. * Added `AggressiveOptimizationOption` to all methods called from `Engine.RunExtraIteration`. * Remove unused `DiagnoserResults.TotalOperations`. * Clarify iteration setup/cleanup timing in xml docs. * Reduce number of tests.
1 parent 7cad308 commit c9dec4d

35 files changed

+439
-410
lines changed

‎src/BenchmarkDotNet/Code/CodeGenerator.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ internal static string Generate(BuildPartition buildPartition)
5656
.Replace("$LoadArguments$", GetLoadArguments(benchmark))
5757
.Replace("$PassArguments$", passArguments)
5858
.Replace("$EngineFactoryType$", GetEngineFactoryTypeName(benchmark))
59-
.Replace("$MeasureExtraStats$", buildInfo.Config.HasExtraStatsDiagnoser() ? "true" : "false")
59+
.Replace("$RunExtraIteration$", buildInfo.Config.HasExtraIterationDiagnoser(benchmark) ? "true" : "false")
6060
.Replace("$DisassemblerEntryMethodName$", DisassemblerConstants.DisassemblerEntryMethodName)
6161
.Replace("$WorkloadMethodCall$", provider.GetWorkloadMethodCall(passArguments))
6262
.Replace("$InProcessDiagnoserRouters$", GetInProcessDiagnoserRouters(buildInfo))

‎src/BenchmarkDotNet/Configs/ImmutableConfig.cs‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -114,19 +114,15 @@ internal ImmutableConfig(
114114

115115
public bool HasMemoryDiagnoser() => diagnosers.OfType<MemoryDiagnoser>().Any();
116116

117-
public bool HasThreadingDiagnoser() => diagnosers.Contains(ThreadingDiagnoser.Default);
118-
119-
public bool HasExceptionDiagnoser() => diagnosers.Contains(ExceptionDiagnoser.Default);
120-
121117
internal bool HasPerfCollectProfiler() => diagnosers.OfType<PerfCollectProfiler>().Any();
122118

123119
internal bool HasDisassemblyDiagnoser() => diagnosers.OfType<DisassemblyDiagnoser>().Any();
124120

125-
public bool HasExtraStatsDiagnoser() => HasMemoryDiagnoser() || HasThreadingDiagnoser() || HasExceptionDiagnoser();
121+
public bool HasExtraIterationDiagnoser(BenchmarkCase benchmarkCase) => HasMemoryDiagnoser() || diagnosers.Any(d => d.GetRunMode(benchmarkCase) == RunMode.ExtraIteration);
126122

127-
public IDiagnoser? GetCompositeDiagnoser(BenchmarkCase benchmarkCase, RunMode runMode)
123+
public IDiagnoser? GetCompositeDiagnoser(BenchmarkCase benchmarkCase, Func<RunMode, bool> runModeComparer)
128124
{
129-
var diagnosersForGivenMode = diagnosers.Where(diagnoser => diagnoser.GetRunMode(benchmarkCase) == runMode).ToImmutableHashSet();
125+
var diagnosersForGivenMode = diagnosers.Where(diagnoser => runModeComparer(diagnoser.GetRunMode(benchmarkCase))).ToImmutableHashSet();
130126

131127
return diagnosersForGivenMode.Any() ? new CompositeDiagnoser(diagnosersForGivenMode) : null;
132128
}

‎src/BenchmarkDotNet/Diagnosers/CompositeDiagnoser.cs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,12 @@
33
using System.Collections.Immutable;
44
using System.ComponentModel;
55
using System.Linq;
6+
using System.Runtime.CompilerServices;
67
using BenchmarkDotNet.Analysers;
78
using BenchmarkDotNet.Engines;
89
using BenchmarkDotNet.Exporters;
910
using BenchmarkDotNet.Loggers;
11+
using BenchmarkDotNet.Portability;
1012
using BenchmarkDotNet.Reports;
1113
using BenchmarkDotNet.Running;
1214
using BenchmarkDotNet.Validators;
@@ -33,6 +35,7 @@ public IEnumerable<IExporter> Exporters
3335
public IEnumerable<IAnalyser> Analysers
3436
=> diagnosers.SelectMany(diagnoser => diagnoser.Analysers);
3537

38+
[MethodImpl(CodeGenHelper.AggressiveOptimizationOption)]
3639
public void Handle(HostSignal signal, DiagnoserActionParameters parameters)
3740
{
3841
foreach (var diagnoser in diagnosers)
@@ -71,6 +74,7 @@ public void DeserializeResults(int index, BenchmarkCase benchmarkCase, string re
7174
[EditorBrowsable(EditorBrowsableState.Never)]
7275
public sealed class CompositeInProcessDiagnoserHandler(IReadOnlyList<InProcessDiagnoserRouter> routers, IHost host, RunMode runMode, InProcessDiagnoserActionArgs parameters)
7376
{
77+
[MethodImpl(CodeGenHelper.AggressiveOptimizationOption)]
7478
public void Handle(BenchmarkSignal signal)
7579
{
7680
if (runMode == RunMode.None)
@@ -80,7 +84,7 @@ public void Handle(BenchmarkSignal signal)
8084

8185
foreach (var router in routers)
8286
{
83-
if (router.runMode == runMode)
87+
if (router.ShouldHandle(runMode))
8488
{
8589
router.handler.Handle(signal, parameters);
8690
}
@@ -93,7 +97,7 @@ public void Handle(BenchmarkSignal signal)
9397

9498
foreach (var router in routers)
9599
{
96-
if (router.runMode != runMode)
100+
if (!router.ShouldHandle(runMode))
97101
{
98102
continue;
99103
}

‎src/BenchmarkDotNet/Diagnosers/DiagnoserResults.cs‎

Lines changed: 5 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -3,35 +3,17 @@
33
using BenchmarkDotNet.Running;
44
using BenchmarkDotNet.Toolchains.Results;
55
using System.Collections.Generic;
6-
using System.Linq;
76

87
namespace BenchmarkDotNet.Diagnosers
98
{
10-
public class DiagnoserResults
9+
public class DiagnoserResults(BenchmarkCase benchmarkCase, ExecuteResult executeResult, BuildResult buildResult)
1110
{
12-
public DiagnoserResults(BenchmarkCase benchmarkCase, ExecuteResult executeResult, BuildResult buildResult)
13-
{
14-
BenchmarkCase = benchmarkCase;
15-
TotalOperations = executeResult.Measurements.Where(measurement => measurement.IsWorkload()).Sum(m => m.Operations);
16-
GcStats = executeResult.GcStats;
17-
ThreadingStats = executeResult.ThreadingStats;
18-
BuildResult = buildResult;
19-
ExceptionFrequency = executeResult.ExceptionFrequency;
20-
Measurements = executeResult.Measurements;
21-
}
11+
public BenchmarkCase BenchmarkCase { get; } = benchmarkCase;
2212

23-
public BenchmarkCase BenchmarkCase { get; }
13+
public GcStats GcStats { get; } = executeResult.GcStats;
2414

25-
public long TotalOperations { get; }
15+
public BuildResult BuildResult { get; } = buildResult;
2616

27-
public GcStats GcStats { get; }
28-
29-
public ThreadingStats ThreadingStats { get; }
30-
31-
public double ExceptionFrequency { get; }
32-
33-
public BuildResult BuildResult { get; }
34-
35-
public IReadOnlyList<Measurement> Measurements { get; }
17+
public IReadOnlyList<Measurement> Measurements { get; } = executeResult.Measurements;
3618
}
3719
}

‎src/BenchmarkDotNet/Diagnosers/ExceptionDiagnoser.cs‎

Lines changed: 60 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -4,50 +4,61 @@
44
using BenchmarkDotNet.Engines;
55
using BenchmarkDotNet.Exporters;
66
using BenchmarkDotNet.Loggers;
7+
using BenchmarkDotNet.Portability;
78
using BenchmarkDotNet.Reports;
89
using BenchmarkDotNet.Running;
910
using BenchmarkDotNet.Validators;
11+
using JetBrains.Annotations;
1012
using System;
1113
using System.Collections.Generic;
14+
using System.ComponentModel;
1215
using System.Linq;
16+
using System.Runtime.CompilerServices;
17+
using System.Runtime.ExceptionServices;
18+
using System.Threading;
1319

1420
namespace BenchmarkDotNet.Diagnosers
1521
{
16-
public class ExceptionDiagnoser : IDiagnoser
22+
public class ExceptionDiagnoser(ExceptionDiagnoserConfig config) : IInProcessDiagnoser
1723
{
18-
public static readonly ExceptionDiagnoser Default = new ExceptionDiagnoser(new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue: true));
24+
public static readonly ExceptionDiagnoser Default = new(new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue: true));
1925

20-
public ExceptionDiagnoser(ExceptionDiagnoserConfig config) => Config = config;
26+
private readonly Dictionary<BenchmarkCase, long> results = [];
2127

22-
public ExceptionDiagnoserConfig Config { get; }
28+
public ExceptionDiagnoserConfig Config { get; } = config;
2329

24-
public IEnumerable<string> Ids => new[] { nameof(ExceptionDiagnoser) };
30+
public IEnumerable<string> Ids => [nameof(ExceptionDiagnoser)];
2531

26-
public IEnumerable<IExporter> Exporters => Array.Empty<IExporter>();
32+
public IEnumerable<IExporter> Exporters => [];
2733

28-
public IEnumerable<IAnalyser> Analysers => Array.Empty<IAnalyser>();
34+
public IEnumerable<IAnalyser> Analysers => [];
2935

3036
public void DisplayResults(ILogger logger) { }
3137

32-
public RunMode GetRunMode(BenchmarkCase benchmarkCase) => RunMode.NoOverhead;
38+
public RunMode GetRunMode(BenchmarkCase benchmarkCase) => RunMode.ExtraIteration;
3339

40+
[MethodImpl(CodeGenHelper.AggressiveOptimizationOption)]
3441
public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { }
3542

36-
public IEnumerable<Metric> ProcessResults(DiagnoserResults results)
43+
public IEnumerable<Metric> ProcessResults(DiagnoserResults diagnoserResults)
3744
{
38-
yield return new Metric(new ExceptionsFrequencyMetricDescriptor(Config), results.ExceptionFrequency);
45+
if (results.TryGetValue(diagnoserResults.BenchmarkCase, out var exceptionsCount))
46+
{
47+
double totalOperations = diagnoserResults.Measurements.First(m => m.IterationStage == IterationStage.Extra).Operations;
48+
yield return new Metric(new ExceptionsFrequencyMetricDescriptor(Config), exceptionsCount / totalOperations);
49+
}
3950
}
4051

41-
public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters) => Enumerable.Empty<ValidationError>();
52+
public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters) => [];
4253

43-
internal class ExceptionsFrequencyMetricDescriptor : IMetricDescriptor
44-
{
45-
public ExceptionDiagnoserConfig Config { get; }
46-
public ExceptionsFrequencyMetricDescriptor(ExceptionDiagnoserConfig config = null)
47-
{
48-
Config = config;
49-
}
54+
void IInProcessDiagnoser.DeserializeResults(BenchmarkCase benchmarkCase, string serializedResults)
55+
=> results.Add(benchmarkCase, long.Parse(serializedResults));
5056

57+
InProcessDiagnoserHandlerData IInProcessDiagnoser.GetHandlerData(BenchmarkCase benchmarkCase)
58+
=> new(typeof(ExceptionDiagnoserInProcessHandler), null);
59+
60+
internal class ExceptionsFrequencyMetricDescriptor(ExceptionDiagnoserConfig config) : IMetricDescriptor
61+
{
5162
public string Id => "ExceptionFrequency";
5263
public string DisplayName => Column.Exceptions;
5364
public string Legend => "Exceptions thrown per single operation";
@@ -57,12 +68,39 @@ public ExceptionsFrequencyMetricDescriptor(ExceptionDiagnoserConfig config = nul
5768
public bool TheGreaterTheBetter => false;
5869
public int PriorityInCategory => 0;
5970
public bool GetIsAvailable(Metric metric)
71+
=> config?.DisplayExceptionsIfZeroValue == true || metric.Value > 0;
72+
}
73+
}
74+
75+
[UsedImplicitly]
76+
[EditorBrowsable(EditorBrowsableState.Never)]
77+
public sealed class ExceptionDiagnoserInProcessHandler : IInProcessDiagnoserHandler
78+
{
79+
private long exceptionsCount;
80+
81+
[MethodImpl(CodeGenHelper.AggressiveOptimizationOption)]
82+
void IInProcessDiagnoserHandler.Handle(BenchmarkSignal signal, InProcessDiagnoserActionArgs args)
83+
{
84+
switch (signal)
6085
{
61-
if (Config == null)
62-
return metric.Value > 0;
63-
else
64-
return Config.DisplayExceptionsIfZeroValue || metric.Value > 0;
86+
case BenchmarkSignal.BeforeExtraIteration:
87+
AppDomain.CurrentDomain.FirstChanceException += OnFirstChanceException;
88+
break;
89+
case BenchmarkSignal.AfterExtraIteration:
90+
AppDomain.CurrentDomain.FirstChanceException -= OnFirstChanceException;
91+
break;
6592
}
6693
}
94+
95+
void IInProcessDiagnoserHandler.Initialize(string? serializedConfig) { }
96+
97+
string IInProcessDiagnoserHandler.SerializeResults()
98+
=> exceptionsCount.ToString();
99+
100+
[MethodImpl(CodeGenHelper.AggressiveOptimizationOption)]
101+
private void OnFirstChanceException(object sender, FirstChanceExceptionEventArgs e)
102+
{
103+
Interlocked.Increment(ref exceptionsCount);
104+
}
67105
}
68106
}

‎src/BenchmarkDotNet/Diagnosers/InProcessDiagnoserRouter.cs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
1-
using BenchmarkDotNet.Running;
1+
using BenchmarkDotNet.Portability;
2+
using BenchmarkDotNet.Running;
23
using JetBrains.Annotations;
34
using System;
45
using System.ComponentModel;
6+
using System.Runtime.CompilerServices;
57

68
namespace BenchmarkDotNet.Diagnosers;
79

@@ -33,4 +35,10 @@ internal static InProcessDiagnoserRouter Create(IInProcessDiagnoser diagnoser, B
3335
runMode = diagnoser.GetRunMode(benchmarkCase)
3436
};
3537
}
38+
39+
[MethodImpl(CodeGenHelper.AggressiveOptimizationOption)]
40+
internal readonly bool ShouldHandle(RunMode runMode)
41+
=> this.runMode == runMode
42+
// ExtraIteration is merged with NoOverhead, so we need to check it explicitly.
43+
|| (runMode == RunMode.NoOverhead && this.runMode == RunMode.ExtraIteration);
3644
}

‎src/BenchmarkDotNet/Diagnosers/RunMode.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@ public enum RunMode : byte
1414
/// </summary>
1515
ExtraRun,
1616
/// <summary>
17+
/// needs a single extra iteration of the benchmark
18+
/// </summary>
19+
ExtraIteration,
20+
/// <summary>
1721
/// no overhead, can be executed without extra run
1822
/// </summary>
1923
NoOverhead,

0 commit comments

Comments
 (0)