From 3cb2d96089438ebe146498ac2fa0cb03eab10cb2 Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 14 Jan 2025 03:35:26 -0500 Subject: [PATCH 1/5] Inline engine stages. Apply AggressiveOptimization to engine methods. --- src/BenchmarkDotNet/Engines/Engine.cs | 196 +++++++++++++++++- .../Engines/EngineGeneralStage.cs | 18 +- .../Engines/EnginePilotStage.cs | 16 +- 3 files changed, 203 insertions(+), 27 deletions(-) diff --git a/src/BenchmarkDotNet/Engines/Engine.cs b/src/BenchmarkDotNet/Engines/Engine.cs index 9dc8cc616e..c7b5f62992 100644 --- a/src/BenchmarkDotNet/Engines/Engine.cs +++ b/src/BenchmarkDotNet/Engines/Engine.cs @@ -1,10 +1,13 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Globalization; using System.Linq; using System.Runtime.CompilerServices; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Mathematics; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; using JetBrains.Annotations; @@ -102,8 +105,14 @@ public void Dispose() } } + // AggressiveOptimization forces the method to go straight to tier1 JIT, and will never be re-jitted, + // eliminating tiered JIT as a potential variable in measurements. + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] public RunResults Run() { + // This method is huge, because all stages are inlined. This ensures the stack size + // remains constant for each benchmark invocation, eliminating stack sizes as a potential variable in measurements. + // #1120 var measurements = new List(); measurements.AddRange(jittingMeasurements); @@ -116,23 +125,185 @@ public RunResults Run() { if (Strategy != RunStrategy.Monitoring) { - var pilotStageResult = pilotStage.Run(); - invokeCount = pilotStageResult.PerfectInvocationCount; - measurements.AddRange(pilotStageResult.Measurements); + // Pilot Stage + { + // If InvocationCount is specified, pilot stage should be skipped + if (TargetJob.HasValue(RunMode.InvocationCountCharacteristic)) + { + } + // Here we want to guess "perfect" amount of invocation + else if (TargetJob.HasValue(RunMode.IterationTimeCharacteristic)) + { + // Perfect invocation count + invokeCount = pilotStage.Autocorrect(MinInvokeCount); + + int iterationCounter = 0; + + int downCount = 0; // Amount of iterations where newInvokeCount < invokeCount + while (true) + { + iterationCounter++; + var measurement = RunIteration(new IterationData(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, UnrollFactor)); + measurements.Add(measurement); + double actualIterationTime = measurement.Nanoseconds; + long newInvokeCount = pilotStage.Autocorrect(Math.Max(pilotStage.minInvokeCount, (long) Math.Round(invokeCount * pilotStage.targetIterationTime / actualIterationTime))); + + if (newInvokeCount < invokeCount) + downCount++; + + if (Math.Abs(newInvokeCount - invokeCount) <= 1 || downCount >= 3) + break; + + invokeCount = newInvokeCount; + } + WriteLine(); + } + else + { + // A case where we don't have specific iteration time. + invokeCount = pilotStage.Autocorrect(pilotStage.minInvokeCount); + + int iterationCounter = 0; + while (true) + { + iterationCounter++; + var measurement = RunIteration(new IterationData(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, UnrollFactor)); + measurements.Add(measurement); + double iterationTime = measurement.Nanoseconds; + double operationError = 2.0 * pilotStage.resolution / invokeCount; // An operation error which has arisen due to the Chronometer precision + + // Max acceptable operation error + double operationMaxError1 = iterationTime / invokeCount * pilotStage.maxRelativeError; + double operationMaxError2 = pilotStage.maxAbsoluteError?.Nanoseconds ?? double.MaxValue; + double operationMaxError = Math.Min(operationMaxError1, operationMaxError2); + + bool isFinished = operationError < operationMaxError && iterationTime >= pilotStage.minIterationTime.Nanoseconds; + if (isFinished) + break; + if (invokeCount >= EnginePilotStage.MaxInvokeCount) + break; + + if (UnrollFactor == 1 && invokeCount < EnvironmentResolver.DefaultUnrollFactorForThroughput) + invokeCount += 1; + else + invokeCount *= 2; + } + WriteLine(); + } + } + // End Pilot Stage if (EvaluateOverhead) { - measurements.AddRange(warmupStage.RunOverhead(invokeCount, UnrollFactor)); - measurements.AddRange(actualStage.RunOverhead(invokeCount, UnrollFactor)); + // Warmup Overhead + { + var warmupMeasurements = new List(); + + var criteria = DefaultStoppingCriteriaFactory.Instance.CreateWarmup(TargetJob, Resolver, IterationMode.Overhead, RunStrategy.Throughput); + int iterationCounter = 0; + while (!criteria.Evaluate(warmupMeasurements).IsFinished) + { + iterationCounter++; + warmupMeasurements.Add(RunIteration(new IterationData(IterationMode.Overhead, IterationStage.Warmup, iterationCounter, invokeCount, UnrollFactor))); + } + WriteLine(); + + measurements.AddRange(warmupMeasurements); + } + // End Warmup Overhead + + // Actual Overhead + { + var measurementsForStatistics = new List(actualStage.maxIterationCount); + + int iterationCounter = 0; + double effectiveMaxRelativeError = EngineActualStage.MaxOverheadRelativeError; + while (true) + { + iterationCounter++; + var measurement = RunIteration(new IterationData(IterationMode.Overhead, IterationStage.Actual, iterationCounter, invokeCount, UnrollFactor)); + measurements.Add(measurement); + measurementsForStatistics.Add(measurement); + + var statistics = MeasurementsStatistics.Calculate(measurementsForStatistics, actualStage.outlierMode); + double actualError = statistics.LegacyConfidenceInterval.Margin; + + double maxError1 = effectiveMaxRelativeError * statistics.Mean; + double maxError2 = actualStage.maxAbsoluteError?.Nanoseconds ?? double.MaxValue; + double maxError = Math.Min(maxError1, maxError2); + + if (iterationCounter >= actualStage.minIterationCount && actualError < maxError) + break; + + if (iterationCounter >= actualStage.maxIterationCount || iterationCounter >= EngineActualStage.MaxOverheadIterationCount) + break; + } + WriteLine(); + } + // End Actual Overhead } } - measurements.AddRange(warmupStage.RunWorkload(invokeCount, UnrollFactor, Strategy)); + // Warmup Workload + { + var workloadMeasurements = new List(); + + var criteria = DefaultStoppingCriteriaFactory.Instance.CreateWarmup(TargetJob, Resolver, IterationMode.Workload, Strategy); + int iterationCounter = 0; + while (!criteria.Evaluate(workloadMeasurements).IsFinished) + { + iterationCounter++; + workloadMeasurements.Add(RunIteration(new IterationData(IterationMode.Workload, IterationStage.Warmup, iterationCounter, invokeCount, UnrollFactor))); + } + WriteLine(); + + measurements.AddRange(workloadMeasurements); + } + // End Warmup Workload } Host.BeforeMainRun(); - measurements.AddRange(actualStage.RunWorkload(invokeCount, UnrollFactor, forceSpecific: Strategy == RunStrategy.Monitoring)); + // Actual Workload + { + if (actualStage.iterationCount == null && Strategy != RunStrategy.Monitoring) + { + // RunAuto + var measurementsForStatistics = new List(actualStage.maxIterationCount); + + int iterationCounter = 0; + double effectiveMaxRelativeError = actualStage.maxRelativeError; + while (true) + { + iterationCounter++; + var measurement = RunIteration(new IterationData(IterationMode.Workload, IterationStage.Actual, iterationCounter, invokeCount, UnrollFactor)); + measurements.Add(measurement); + measurementsForStatistics.Add(measurement); + + var statistics = MeasurementsStatistics.Calculate(measurementsForStatistics, actualStage.outlierMode); + double actualError = statistics.LegacyConfidenceInterval.Margin; + + double maxError1 = effectiveMaxRelativeError * statistics.Mean; + double maxError2 = actualStage.maxAbsoluteError?.Nanoseconds ?? double.MaxValue; + double maxError = Math.Min(maxError1, maxError2); + + if (iterationCounter >= actualStage.minIterationCount && actualError < maxError) + break; + + if (iterationCounter >= actualStage.maxIterationCount) + break; + } + } + else + { + // RunSpecific + var iterationCount = actualStage.iterationCount ?? EngineActualStage.DefaultWorkloadCount; + for (int i = 0; i < iterationCount; i++) + measurements.Add(RunIteration(new IterationData(IterationMode.Workload, IterationStage.Actual, i + 1, invokeCount, UnrollFactor))); + } + WriteLine(); + } + // End Actual Workload Host.AfterMainRun(); @@ -148,11 +319,15 @@ public RunResults Run() return new RunResults(measurements, outlierMode, workGcHasDone, threadingStats, exceptionFrequency); } + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] public Measurement RunIteration(IterationData data) { // Initialization long invokeCount = data.InvokeCount; int unrollFactor = data.UnrollFactor; + if (invokeCount % unrollFactor != 0) + throw new ArgumentOutOfRangeException($"InvokeCount({invokeCount}) should be a multiple of UnrollFactor({unrollFactor})."); + long totalOperations = invokeCount * OperationsPerInvoke; bool isOverhead = data.IterationMode == IterationMode.Overhead; bool randomizeMemory = !isOverhead && MemoryRandomization; @@ -167,7 +342,7 @@ public Measurement RunIteration(IterationData data) EngineEventSource.Log.IterationStart(data.IterationMode, data.IterationStage, totalOperations); var clockSpan = randomizeMemory - ? MeasureWithRandomMemory(action, invokeCount / unrollFactor) + ? MeasureWithRandomStack(action, invokeCount / unrollFactor) : Measure(action, invokeCount / unrollFactor); if (EngineEventSource.Log.IsEnabled()) @@ -193,8 +368,8 @@ public Measurement RunIteration(IterationData data) // This is in a separate method, because stackalloc can affect code alignment, // resulting in unexpected measurements on some AMD cpus, // even if the stackalloc branch isn't executed. (#2366) - [MethodImpl(MethodImplOptions.NoInlining)] - private unsafe ClockSpan MeasureWithRandomMemory(Action action, long invokeCount) + [MethodImpl(MethodImplOptions.NoInlining | CodeGenHelper.AggressiveOptimizationOption)] + private unsafe ClockSpan MeasureWithRandomStack(Action action, long invokeCount) { byte* stackMemory = stackalloc byte[random.Next(32)]; var clockSpan = Measure(action, invokeCount); @@ -205,6 +380,7 @@ private unsafe ClockSpan MeasureWithRandomMemory(Action action, long invok [MethodImpl(MethodImplOptions.NoInlining)] private unsafe void Consume(byte* _) { } + [MethodImpl(MethodImplOptions.NoInlining | CodeGenHelper.AggressiveOptimizationOption)] private ClockSpan Measure(Action action, long invokeCount) { var clock = Clock.Start(); diff --git a/src/BenchmarkDotNet/Engines/EngineGeneralStage.cs b/src/BenchmarkDotNet/Engines/EngineGeneralStage.cs index 6394db3d1e..c3acd3c58a 100644 --- a/src/BenchmarkDotNet/Engines/EngineGeneralStage.cs +++ b/src/BenchmarkDotNet/Engines/EngineGeneralStage.cs @@ -11,15 +11,15 @@ namespace BenchmarkDotNet.Engines public class EngineActualStage : EngineStage { internal const int MaxOverheadIterationCount = 20; - private const double MaxOverheadRelativeError = 0.05; - private const int DefaultWorkloadCount = 10; - - private readonly int? iterationCount; - private readonly double maxRelativeError; - private readonly TimeInterval? maxAbsoluteError; - private readonly OutlierMode outlierMode; - private readonly int minIterationCount; - private readonly int maxIterationCount; + internal const double MaxOverheadRelativeError = 0.05; + internal const int DefaultWorkloadCount = 10; + + internal readonly int? iterationCount; + internal readonly double maxRelativeError; + internal readonly TimeInterval? maxAbsoluteError; + internal readonly OutlierMode outlierMode; + internal readonly int minIterationCount; + internal readonly int maxIterationCount; public EngineActualStage(IEngine engine) : base(engine) { diff --git a/src/BenchmarkDotNet/Engines/EnginePilotStage.cs b/src/BenchmarkDotNet/Engines/EnginePilotStage.cs index 6adf6d964f..7ededb15af 100644 --- a/src/BenchmarkDotNet/Engines/EnginePilotStage.cs +++ b/src/BenchmarkDotNet/Engines/EnginePilotStage.cs @@ -30,13 +30,13 @@ public PilotStageResult(long perfectInvocationCount) internal const long MaxInvokeCount = (long.MaxValue / 2 + 1) / 2; - private readonly int unrollFactor; - private readonly TimeInterval minIterationTime; - private readonly int minInvokeCount; - private readonly double maxRelativeError; - private readonly TimeInterval? maxAbsoluteError; - private readonly double targetIterationTime; - private readonly double resolution; + internal readonly int unrollFactor; + internal readonly TimeInterval minIterationTime; + internal readonly int minInvokeCount; + internal readonly double maxRelativeError; + internal readonly TimeInterval? maxAbsoluteError; + internal readonly double targetIterationTime; + internal readonly double resolution; public EnginePilotStage(IEngine engine) : base(engine) { @@ -132,6 +132,6 @@ private PilotStageResult RunSpecific() return new PilotStageResult(invokeCount, measurements); } - private long Autocorrect(long count) => (count + unrollFactor - 1) / unrollFactor * unrollFactor; + internal long Autocorrect(long count) => (count + unrollFactor - 1) / unrollFactor * unrollFactor; } } \ No newline at end of file From 284644fcc99dabe636e2a7b2a8e4d470e21ba922 Mon Sep 17 00:00:00 2001 From: Tim Cassell Date: Mon, 3 Feb 2025 13:15:23 -0500 Subject: [PATCH 2/5] Update src/BenchmarkDotNet/Engines/Engine.cs Co-authored-by: Corniel Nobel --- src/BenchmarkDotNet/Engines/Engine.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Engines/Engine.cs b/src/BenchmarkDotNet/Engines/Engine.cs index c7b5f62992..dc2217a904 100644 --- a/src/BenchmarkDotNet/Engines/Engine.cs +++ b/src/BenchmarkDotNet/Engines/Engine.cs @@ -326,7 +326,7 @@ public Measurement RunIteration(IterationData data) long invokeCount = data.InvokeCount; int unrollFactor = data.UnrollFactor; if (invokeCount % unrollFactor != 0) - throw new ArgumentOutOfRangeException($"InvokeCount({invokeCount}) should be a multiple of UnrollFactor({unrollFactor})."); + throw new ArgumentOutOfRangeException(nameof(data), $"InvokeCount({invokeCount}) should be a multiple of UnrollFactor({unrollFactor})."); long totalOperations = invokeCount * OperationsPerInvoke; bool isOverhead = data.IterationMode == IterationMode.Overhead; From 091bb56b4f6388493b02857a2cc71ea0ca71aacf Mon Sep 17 00:00:00 2001 From: Tim Cassell Date: Sat, 15 Feb 2025 11:48:16 -0500 Subject: [PATCH 3/5] Refactor to use `IEngineStageEvaluator` for constant instruction location and simpler engine code. --- src/BenchmarkDotNet/Engines/Engine.cs | 229 ++++-------------- ...neGeneralStage.cs => EngineActualStage.cs} | 74 +++++- .../Engines/EnginePilotStage.cs | 98 +++++++- .../Engines/EngineWarmupStage.cs | 14 ++ .../Engines/IEngineStageEvaluator.cs | 11 + 5 files changed, 225 insertions(+), 201 deletions(-) rename src/BenchmarkDotNet/Engines/{EngineGeneralStage.cs => EngineActualStage.cs} (57%) create mode 100644 src/BenchmarkDotNet/Engines/IEngineStageEvaluator.cs diff --git a/src/BenchmarkDotNet/Engines/Engine.cs b/src/BenchmarkDotNet/Engines/Engine.cs index dc2217a904..c665a9de26 100644 --- a/src/BenchmarkDotNet/Engines/Engine.cs +++ b/src/BenchmarkDotNet/Engines/Engine.cs @@ -105,14 +105,41 @@ public void Dispose() } } + [MethodImpl(MethodImplOptions.NoInlining)] + private IEnumerable<(IterationStage stage, IterationMode mode, IEngineStageEvaluator evaluator)> EnumerateStages() + { + if (Strategy != RunStrategy.ColdStart) + { + if (Strategy != RunStrategy.Monitoring) + { + var pilotEvaluator = pilotStage.GetEvaluator(); + if (pilotEvaluator != null) + { + yield return (IterationStage.Pilot, IterationMode.Workload, pilotEvaluator); + } + + if (EvaluateOverhead) + { + yield return (IterationStage.Warmup, IterationMode.Overhead, warmupStage.GetOverheadEvaluator()); + yield return (IterationStage.Actual, IterationMode.Overhead, actualStage.GetOverheadEvaluator()); + } + } + + yield return (IterationStage.Warmup, IterationMode.Workload, warmupStage.GetWorkloadEvaluator(Strategy)); + } + + Host.BeforeMainRun(); + + yield return (IterationStage.Actual, IterationMode.Workload, actualStage.GetWorkloadEvaluator(Strategy == RunStrategy.Monitoring)); + + Host.AfterMainRun(); + } + // AggressiveOptimization forces the method to go straight to tier1 JIT, and will never be re-jitted, // eliminating tiered JIT as a potential variable in measurements. [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] public RunResults Run() { - // This method is huge, because all stages are inlined. This ensures the stack size - // remains constant for each benchmark invocation, eliminating stack sizes as a potential variable in measurements. - // #1120 var measurements = new List(); measurements.AddRange(jittingMeasurements); @@ -121,191 +148,23 @@ public RunResults Run() if (EngineEventSource.Log.IsEnabled()) EngineEventSource.Log.BenchmarkStart(BenchmarkName); - if (Strategy != RunStrategy.ColdStart) - { - if (Strategy != RunStrategy.Monitoring) - { - // Pilot Stage - { - // If InvocationCount is specified, pilot stage should be skipped - if (TargetJob.HasValue(RunMode.InvocationCountCharacteristic)) - { - } - // Here we want to guess "perfect" amount of invocation - else if (TargetJob.HasValue(RunMode.IterationTimeCharacteristic)) - { - // Perfect invocation count - invokeCount = pilotStage.Autocorrect(MinInvokeCount); - - int iterationCounter = 0; - - int downCount = 0; // Amount of iterations where newInvokeCount < invokeCount - while (true) - { - iterationCounter++; - var measurement = RunIteration(new IterationData(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, UnrollFactor)); - measurements.Add(measurement); - double actualIterationTime = measurement.Nanoseconds; - long newInvokeCount = pilotStage.Autocorrect(Math.Max(pilotStage.minInvokeCount, (long) Math.Round(invokeCount * pilotStage.targetIterationTime / actualIterationTime))); - - if (newInvokeCount < invokeCount) - downCount++; - - if (Math.Abs(newInvokeCount - invokeCount) <= 1 || downCount >= 3) - break; - - invokeCount = newInvokeCount; - } - WriteLine(); - } - else - { - // A case where we don't have specific iteration time. - invokeCount = pilotStage.Autocorrect(pilotStage.minInvokeCount); - - int iterationCounter = 0; - while (true) - { - iterationCounter++; - var measurement = RunIteration(new IterationData(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, UnrollFactor)); - measurements.Add(measurement); - double iterationTime = measurement.Nanoseconds; - double operationError = 2.0 * pilotStage.resolution / invokeCount; // An operation error which has arisen due to the Chronometer precision - - // Max acceptable operation error - double operationMaxError1 = iterationTime / invokeCount * pilotStage.maxRelativeError; - double operationMaxError2 = pilotStage.maxAbsoluteError?.Nanoseconds ?? double.MaxValue; - double operationMaxError = Math.Min(operationMaxError1, operationMaxError2); - - bool isFinished = operationError < operationMaxError && iterationTime >= pilotStage.minIterationTime.Nanoseconds; - if (isFinished) - break; - if (invokeCount >= EnginePilotStage.MaxInvokeCount) - break; - - if (UnrollFactor == 1 && invokeCount < EnvironmentResolver.DefaultUnrollFactorForThroughput) - invokeCount += 1; - else - invokeCount *= 2; - } - WriteLine(); - } - } - // End Pilot Stage - - if (EvaluateOverhead) - { - // Warmup Overhead - { - var warmupMeasurements = new List(); - - var criteria = DefaultStoppingCriteriaFactory.Instance.CreateWarmup(TargetJob, Resolver, IterationMode.Overhead, RunStrategy.Throughput); - int iterationCounter = 0; - while (!criteria.Evaluate(warmupMeasurements).IsFinished) - { - iterationCounter++; - warmupMeasurements.Add(RunIteration(new IterationData(IterationMode.Overhead, IterationStage.Warmup, iterationCounter, invokeCount, UnrollFactor))); - } - WriteLine(); - - measurements.AddRange(warmupMeasurements); - } - // End Warmup Overhead - - // Actual Overhead - { - var measurementsForStatistics = new List(actualStage.maxIterationCount); - - int iterationCounter = 0; - double effectiveMaxRelativeError = EngineActualStage.MaxOverheadRelativeError; - while (true) - { - iterationCounter++; - var measurement = RunIteration(new IterationData(IterationMode.Overhead, IterationStage.Actual, iterationCounter, invokeCount, UnrollFactor)); - measurements.Add(measurement); - measurementsForStatistics.Add(measurement); - - var statistics = MeasurementsStatistics.Calculate(measurementsForStatistics, actualStage.outlierMode); - double actualError = statistics.LegacyConfidenceInterval.Margin; - - double maxError1 = effectiveMaxRelativeError * statistics.Mean; - double maxError2 = actualStage.maxAbsoluteError?.Nanoseconds ?? double.MaxValue; - double maxError = Math.Min(maxError1, maxError2); - - if (iterationCounter >= actualStage.minIterationCount && actualError < maxError) - break; - - if (iterationCounter >= actualStage.maxIterationCount || iterationCounter >= EngineActualStage.MaxOverheadIterationCount) - break; - } - WriteLine(); - } - // End Actual Overhead - } - } - - // Warmup Workload - { - var workloadMeasurements = new List(); - - var criteria = DefaultStoppingCriteriaFactory.Instance.CreateWarmup(TargetJob, Resolver, IterationMode.Workload, Strategy); - int iterationCounter = 0; - while (!criteria.Evaluate(workloadMeasurements).IsFinished) - { - iterationCounter++; - workloadMeasurements.Add(RunIteration(new IterationData(IterationMode.Workload, IterationStage.Warmup, iterationCounter, invokeCount, UnrollFactor))); - } - WriteLine(); - - measurements.AddRange(workloadMeasurements); - } - // End Warmup Workload - } - - Host.BeforeMainRun(); + // Enumerate the stages and run iterations in a loop to ensure each benchmark invocation is called with a constant stack size. + // #1120 + foreach (var (stage, mode, evaluator) in EnumerateStages()) + { + var stageMeasurements = new List(evaluator.MaxIterationCount); + int iterationCounter = 0; + while (!evaluator.EvaluateShouldStop(stageMeasurements, ref invokeCount)) + { + // TODO: Not sure why index is 1-based? 0-based is standard. + ++iterationCounter; + var measurement = RunIteration(new IterationData(mode, stage, iterationCounter, invokeCount, UnrollFactor)); + stageMeasurements.Add(measurement); + } + measurements.AddRange(stageMeasurements); - // Actual Workload - { - if (actualStage.iterationCount == null && Strategy != RunStrategy.Monitoring) - { - // RunAuto - var measurementsForStatistics = new List(actualStage.maxIterationCount); - - int iterationCounter = 0; - double effectiveMaxRelativeError = actualStage.maxRelativeError; - while (true) - { - iterationCounter++; - var measurement = RunIteration(new IterationData(IterationMode.Workload, IterationStage.Actual, iterationCounter, invokeCount, UnrollFactor)); - measurements.Add(measurement); - measurementsForStatistics.Add(measurement); - - var statistics = MeasurementsStatistics.Calculate(measurementsForStatistics, actualStage.outlierMode); - double actualError = statistics.LegacyConfidenceInterval.Margin; - - double maxError1 = effectiveMaxRelativeError * statistics.Mean; - double maxError2 = actualStage.maxAbsoluteError?.Nanoseconds ?? double.MaxValue; - double maxError = Math.Min(maxError1, maxError2); - - if (iterationCounter >= actualStage.minIterationCount && actualError < maxError) - break; - - if (iterationCounter >= actualStage.maxIterationCount) - break; - } - } - else - { - // RunSpecific - var iterationCount = actualStage.iterationCount ?? EngineActualStage.DefaultWorkloadCount; - for (int i = 0; i < iterationCount; i++) - measurements.Add(RunIteration(new IterationData(IterationMode.Workload, IterationStage.Actual, i + 1, invokeCount, UnrollFactor))); - } WriteLine(); } - // End Actual Workload - - Host.AfterMainRun(); (GcStats workGcHasDone, ThreadingStats threadingStats, double exceptionFrequency) = includeExtraStats ? GetExtraStats(new IterationData(IterationMode.Workload, IterationStage.Actual, 0, invokeCount, UnrollFactor)) diff --git a/src/BenchmarkDotNet/Engines/EngineGeneralStage.cs b/src/BenchmarkDotNet/Engines/EngineActualStage.cs similarity index 57% rename from src/BenchmarkDotNet/Engines/EngineGeneralStage.cs rename to src/BenchmarkDotNet/Engines/EngineActualStage.cs index c3acd3c58a..2f6d4018f9 100644 --- a/src/BenchmarkDotNet/Engines/EngineGeneralStage.cs +++ b/src/BenchmarkDotNet/Engines/EngineActualStage.cs @@ -11,15 +11,15 @@ namespace BenchmarkDotNet.Engines public class EngineActualStage : EngineStage { internal const int MaxOverheadIterationCount = 20; - internal const double MaxOverheadRelativeError = 0.05; - internal const int DefaultWorkloadCount = 10; + private const double MaxOverheadRelativeError = 0.05; + private const int DefaultWorkloadCount = 10; - internal readonly int? iterationCount; - internal readonly double maxRelativeError; - internal readonly TimeInterval? maxAbsoluteError; - internal readonly OutlierMode outlierMode; - internal readonly int minIterationCount; - internal readonly int maxIterationCount; + private readonly int? iterationCount; + private readonly double maxRelativeError; + private readonly TimeInterval? maxAbsoluteError; + private readonly OutlierMode outlierMode; + private readonly int minIterationCount; + private readonly int maxIterationCount; public EngineActualStage(IEngine engine) : base(engine) { @@ -86,5 +86,63 @@ private List RunSpecific(long invokeCount, IterationMode iterationM return measurements; } + + internal IEngineStageEvaluator GetOverheadEvaluator() + => new AutoEvaluator(this, true); + + internal IEngineStageEvaluator GetWorkloadEvaluator(bool forceSpecific) + => iterationCount == null && !forceSpecific + ? new AutoEvaluator(this, false) + : new SpecificEvaluator(this); + + private sealed class AutoEvaluator(EngineActualStage stage, bool isOverhead) : IEngineStageEvaluator + { + public int MaxIterationCount => stage.maxIterationCount; + + private readonly List _measurementsForStatistics = new (stage.maxIterationCount); + private int _iterationCounter = 0; + + public bool EvaluateShouldStop(List measurements, ref long invokeCount) + { + if (measurements.Count == 0) + { + return false; + } + + double effectiveMaxRelativeError = isOverhead ? MaxOverheadRelativeError : stage.maxRelativeError; + _iterationCounter++; + var measurement = measurements[measurements.Count - 1]; + _measurementsForStatistics.Add(measurement); + + var statistics = MeasurementsStatistics.Calculate(_measurementsForStatistics, stage.outlierMode); + double actualError = statistics.LegacyConfidenceInterval.Margin; + + double maxError1 = effectiveMaxRelativeError * statistics.Mean; + double maxError2 = stage.maxAbsoluteError?.Nanoseconds ?? double.MaxValue; + double maxError = Math.Min(maxError1, maxError2); + + if (_iterationCounter >= stage.minIterationCount && actualError < maxError) + { + return true; + } + + if (_iterationCounter >= stage.maxIterationCount || isOverhead && _iterationCounter >= MaxOverheadIterationCount) + { + return true; + } + + return false; + } + } + + private sealed class SpecificEvaluator(EngineActualStage stage) : IEngineStageEvaluator + { + public int MaxIterationCount => stage.iterationCount ?? DefaultWorkloadCount; + + private int _iterationCount = 0; + + public bool EvaluateShouldStop(List measurements, ref long invokeCount) + => ++_iterationCount > MaxIterationCount; + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/EnginePilotStage.cs b/src/BenchmarkDotNet/Engines/EnginePilotStage.cs index 7ededb15af..03d079443e 100644 --- a/src/BenchmarkDotNet/Engines/EnginePilotStage.cs +++ b/src/BenchmarkDotNet/Engines/EnginePilotStage.cs @@ -30,13 +30,13 @@ public PilotStageResult(long perfectInvocationCount) internal const long MaxInvokeCount = (long.MaxValue / 2 + 1) / 2; - internal readonly int unrollFactor; - internal readonly TimeInterval minIterationTime; - internal readonly int minInvokeCount; - internal readonly double maxRelativeError; - internal readonly TimeInterval? maxAbsoluteError; - internal readonly double targetIterationTime; - internal readonly double resolution; + private readonly int unrollFactor; + private readonly TimeInterval minIterationTime; + private readonly int minInvokeCount; + private readonly double maxRelativeError; + private readonly TimeInterval? maxAbsoluteError; + private readonly double targetIterationTime; + private readonly double resolution; public EnginePilotStage(IEngine engine) : base(engine) { @@ -132,6 +132,88 @@ private PilotStageResult RunSpecific() return new PilotStageResult(invokeCount, measurements); } - internal long Autocorrect(long count) => (count + unrollFactor - 1) / unrollFactor * unrollFactor; + private long Autocorrect(long count) => (count + unrollFactor - 1) / unrollFactor * unrollFactor; + + internal IEngineStageEvaluator GetEvaluator() + { + // If InvocationCount is specified, pilot stage should be skipped + return TargetJob.HasValue(RunMode.InvocationCountCharacteristic) ? null + // Here we want to guess "perfect" amount of invocation + : TargetJob.HasValue(RunMode.IterationTimeCharacteristic) ? new SpecificEvaluator(this) + : new AutoEvaluator(this); + } + + private sealed class AutoEvaluator(EnginePilotStage stage) : IEngineStageEvaluator + { + public int MaxIterationCount => 0; + + public bool EvaluateShouldStop(List measurements, ref long invokeCount) + { + if (measurements.Count == 0) + { + invokeCount = stage.Autocorrect(stage.minInvokeCount); + return false; + } + + var measurement = measurements[measurements.Count - 1]; + double iterationTime = measurement.Nanoseconds; + double operationError = 2.0 * stage.resolution / invokeCount; // An operation error which has arisen due to the Chronometer precision + + // Max acceptable operation error + double operationMaxError1 = iterationTime / invokeCount * stage.maxRelativeError; + double operationMaxError2 = stage.maxAbsoluteError?.Nanoseconds ?? double.MaxValue; + double operationMaxError = Math.Min(operationMaxError1, operationMaxError2); + + bool isFinished = operationError < operationMaxError && iterationTime >= stage.minIterationTime.Nanoseconds; + if (isFinished || invokeCount >= MaxInvokeCount) + { + return true; + } + + if (stage.unrollFactor == 1 && invokeCount < EnvironmentResolver.DefaultUnrollFactorForThroughput) + { + ++invokeCount; + } + else + { + invokeCount *= 2; + } + + return false; + } + } + + private sealed class SpecificEvaluator(EnginePilotStage stage) : IEngineStageEvaluator + { + public int MaxIterationCount => 0; + + private int _downCount = 0; // Amount of iterations where newInvokeCount < invokeCount + + public bool EvaluateShouldStop(List measurements, ref long invokeCount) + { + if (measurements.Count == 0) + { + invokeCount = stage.Autocorrect(Engine.MinInvokeCount); + return false; + } + + var measurement = measurements[measurements.Count - 1]; + double actualIterationTime = measurement.Nanoseconds; + long newInvokeCount = stage.Autocorrect(Math.Max(stage.minInvokeCount, (long) Math.Round(invokeCount * stage.targetIterationTime / actualIterationTime))); + + if (newInvokeCount < invokeCount) + { + _downCount++; + } + + if (Math.Abs(newInvokeCount - invokeCount) <= 1 || _downCount >= 3) + { + return true; + } + + invokeCount = newInvokeCount; + return false; + } + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs b/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs index 08ec7ec729..12c5e964ba 100644 --- a/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs +++ b/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs @@ -20,5 +20,19 @@ internal IReadOnlyList Run(long invokeCount, IterationMode iteratio var criteria = DefaultStoppingCriteriaFactory.Instance.CreateWarmup(engine.TargetJob, engine.Resolver, iterationMode, runStrategy); return Run(criteria, invokeCount, iterationMode, IterationStage.Warmup, unrollFactor); } + + internal IEngineStageEvaluator GetOverheadEvaluator() + => new Evaluator(DefaultStoppingCriteriaFactory.Instance.CreateWarmup(engine.TargetJob, engine.Resolver, IterationMode.Overhead, RunStrategy.Throughput)); + + internal IEngineStageEvaluator GetWorkloadEvaluator(RunStrategy runStrategy) + => new Evaluator(DefaultStoppingCriteriaFactory.Instance.CreateWarmup(engine.TargetJob, engine.Resolver, IterationMode.Workload, runStrategy)); + + private sealed class Evaluator(IStoppingCriteria stoppingCriteria) : IEngineStageEvaluator + { + public int MaxIterationCount => stoppingCriteria.MaxIterationCount; + + public bool EvaluateShouldStop(List measurements, ref long invokeCount) + => stoppingCriteria.Evaluate(measurements).IsFinished; + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/IEngineStageEvaluator.cs b/src/BenchmarkDotNet/Engines/IEngineStageEvaluator.cs new file mode 100644 index 0000000000..b2eaa5316f --- /dev/null +++ b/src/BenchmarkDotNet/Engines/IEngineStageEvaluator.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; +using BenchmarkDotNet.Reports; + +namespace BenchmarkDotNet.Engines +{ + internal interface IEngineStageEvaluator + { + bool EvaluateShouldStop(List measurements, ref long invokeCount); + int MaxIterationCount { get; } + } +} \ No newline at end of file From 38466481bf316c0b9c2878525e26c4532d063402 Mon Sep 17 00:00:00 2001 From: Tim Cassell Date: Sat, 3 May 2025 20:09:32 -0400 Subject: [PATCH 4/5] Refactored according to PR comments. --- src/BenchmarkDotNet/Engines/Engine.cs | 65 ++-- .../Engines/EngineActualStage.cs | 185 ++++-------- .../Engines/EnginePilotStage.cs | 282 ++++++------------ src/BenchmarkDotNet/Engines/EngineStage.cs | 61 ++-- .../Engines/EngineWarmupStage.cs | 82 +++-- .../Engines/IEngineStageEvaluator.cs | 11 - .../AutoWarmupStoppingCriteria.cs | 97 ------ .../DefaultStoppingCriteriaFactory.cs | 43 --- .../StoppingCriteria/FixedStoppingCriteria.cs | 37 --- .../StoppingCriteria/IStoppingCriteria.cs | 33 -- .../StoppingCriteria/StoppingCriteriaBase.cs | 33 -- .../StoppingCriteria/StoppingResult.cs | 18 -- ...tageTests.cs => EngineActualStageTests.cs} | 13 +- .../Engine/EnginePilotStageTests.cs | 16 +- .../Engine/EngineWarmupStageTests.cs | 15 +- .../AutoWarmupStoppingCriteriaTests.cs | 80 ----- .../FixedStoppingCriteriaTests.cs | 67 ----- .../StoppingCriteriaTestsBase.cs | 73 ----- .../BenchmarkDotNet.Tests/Mocks/MockEngine.cs | 14 + 19 files changed, 286 insertions(+), 939 deletions(-) delete mode 100644 src/BenchmarkDotNet/Engines/IEngineStageEvaluator.cs delete mode 100644 src/BenchmarkDotNet/Engines/StoppingCriteria/AutoWarmupStoppingCriteria.cs delete mode 100644 src/BenchmarkDotNet/Engines/StoppingCriteria/DefaultStoppingCriteriaFactory.cs delete mode 100644 src/BenchmarkDotNet/Engines/StoppingCriteria/FixedStoppingCriteria.cs delete mode 100644 src/BenchmarkDotNet/Engines/StoppingCriteria/IStoppingCriteria.cs delete mode 100644 src/BenchmarkDotNet/Engines/StoppingCriteria/StoppingCriteriaBase.cs delete mode 100644 src/BenchmarkDotNet/Engines/StoppingCriteria/StoppingResult.cs rename tests/BenchmarkDotNet.Tests/Engine/{EngineGeneralStageTests.cs => EngineActualStageTests.cs} (85%) delete mode 100644 tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/AutoWarmupStoppingCriteriaTests.cs delete mode 100644 tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/FixedStoppingCriteriaTests.cs delete mode 100644 tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/StoppingCriteriaTestsBase.cs diff --git a/src/BenchmarkDotNet/Engines/Engine.cs b/src/BenchmarkDotNet/Engines/Engine.cs index c665a9de26..859666f125 100644 --- a/src/BenchmarkDotNet/Engines/Engine.cs +++ b/src/BenchmarkDotNet/Engines/Engine.cs @@ -18,8 +18,6 @@ namespace BenchmarkDotNet.Engines [UsedImplicitly] public class Engine : IEngine { - public const int MinInvokeCount = 4; - [PublicAPI] public IHost Host { get; } [PublicAPI] public Action WorkloadAction { get; } [PublicAPI] public Action Dummy1Action { get; } @@ -44,9 +42,6 @@ public class Engine : IEngine private bool MemoryRandomization { get; } private readonly List jittingMeasurements = new (10); - private readonly EnginePilotStage pilotStage; - private readonly EngineWarmupStage warmupStage; - private readonly EngineActualStage actualStage; private readonly bool includeExtraStats; private readonly Random random; @@ -82,10 +77,6 @@ internal Engine( EvaluateOverhead = targetJob.ResolveValue(AccuracyMode.EvaluateOverheadCharacteristic, Resolver); MemoryRandomization = targetJob.ResolveValue(RunMode.MemoryRandomizationCharacteristic, Resolver); - warmupStage = new EngineWarmupStage(this); - pilotStage = new EnginePilotStage(this); - actualStage = new EngineActualStage(this); - random = new Random(12345); // we are using constant seed to try to get repeatable results } @@ -105,36 +96,6 @@ public void Dispose() } } - [MethodImpl(MethodImplOptions.NoInlining)] - private IEnumerable<(IterationStage stage, IterationMode mode, IEngineStageEvaluator evaluator)> EnumerateStages() - { - if (Strategy != RunStrategy.ColdStart) - { - if (Strategy != RunStrategy.Monitoring) - { - var pilotEvaluator = pilotStage.GetEvaluator(); - if (pilotEvaluator != null) - { - yield return (IterationStage.Pilot, IterationMode.Workload, pilotEvaluator); - } - - if (EvaluateOverhead) - { - yield return (IterationStage.Warmup, IterationMode.Overhead, warmupStage.GetOverheadEvaluator()); - yield return (IterationStage.Actual, IterationMode.Overhead, actualStage.GetOverheadEvaluator()); - } - } - - yield return (IterationStage.Warmup, IterationMode.Workload, warmupStage.GetWorkloadEvaluator(Strategy)); - } - - Host.BeforeMainRun(); - - yield return (IterationStage.Actual, IterationMode.Workload, actualStage.GetWorkloadEvaluator(Strategy == RunStrategy.Monitoring)); - - Host.AfterMainRun(); - } - // AggressiveOptimization forces the method to go straight to tier1 JIT, and will never be re-jitted, // eliminating tiered JIT as a potential variable in measurements. [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] @@ -150,20 +111,30 @@ public RunResults Run() // Enumerate the stages and run iterations in a loop to ensure each benchmark invocation is called with a constant stack size. // #1120 - foreach (var (stage, mode, evaluator) in EnumerateStages()) + foreach (var stage in EngineStage.EnumerateStages(this, Strategy, EvaluateOverhead)) { - var stageMeasurements = new List(evaluator.MaxIterationCount); - int iterationCounter = 0; - while (!evaluator.EvaluateShouldStop(stageMeasurements, ref invokeCount)) + if (stage.Stage == IterationStage.Actual && stage.Mode == IterationMode.Workload) { - // TODO: Not sure why index is 1-based? 0-based is standard. - ++iterationCounter; - var measurement = RunIteration(new IterationData(mode, stage, iterationCounter, invokeCount, UnrollFactor)); + Host.BeforeMainRun(); + } + + var stageMeasurements = stage.GetMeasurementList(); + // 1-based iterationIndex + int iterationIndex = 1; + while (stage.GetShouldRunIteration(stageMeasurements, ref invokeCount)) + { + var measurement = RunIteration(new IterationData(stage.Mode, stage.Stage, iterationIndex, invokeCount, UnrollFactor)); stageMeasurements.Add(measurement); + ++iterationIndex; } measurements.AddRange(stageMeasurements); - WriteLine(); + WriteLine(); + + if (stage.Stage == IterationStage.Actual && stage.Mode == IterationMode.Workload) + { + Host.AfterMainRun(); + } } (GcStats workGcHasDone, ThreadingStats threadingStats, double exceptionFrequency) = includeExtraStats diff --git a/src/BenchmarkDotNet/Engines/EngineActualStage.cs b/src/BenchmarkDotNet/Engines/EngineActualStage.cs index 2f6d4018f9..2fbf9206ae 100644 --- a/src/BenchmarkDotNet/Engines/EngineActualStage.cs +++ b/src/BenchmarkDotNet/Engines/EngineActualStage.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Mathematics; using BenchmarkDotNet.Reports; @@ -8,141 +9,75 @@ namespace BenchmarkDotNet.Engines { - public class EngineActualStage : EngineStage + internal abstract class EngineActualStage(IterationMode iterationMode) : EngineStage(IterationStage.Actual, iterationMode) { internal const int MaxOverheadIterationCount = 20; - private const double MaxOverheadRelativeError = 0.05; - private const int DefaultWorkloadCount = 10; - - private readonly int? iterationCount; - private readonly double maxRelativeError; - private readonly TimeInterval? maxAbsoluteError; - private readonly OutlierMode outlierMode; - private readonly int minIterationCount; - private readonly int maxIterationCount; - - public EngineActualStage(IEngine engine) : base(engine) - { - iterationCount = engine.TargetJob.ResolveValueAsNullable(RunMode.IterationCountCharacteristic); - maxRelativeError = engine.TargetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, engine.Resolver); - maxAbsoluteError = engine.TargetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic); - outlierMode = engine.TargetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, engine.Resolver); - minIterationCount = engine.TargetJob.ResolveValue(RunMode.MinIterationCountCharacteristic, engine.Resolver); - maxIterationCount = engine.TargetJob.ResolveValue(RunMode.MaxIterationCountCharacteristic, engine.Resolver); - } - - public IReadOnlyList RunOverhead(long invokeCount, int unrollFactor) - => RunAuto(invokeCount, IterationMode.Overhead, unrollFactor); - - public IReadOnlyList RunWorkload(long invokeCount, int unrollFactor, bool forceSpecific = false) - => Run(invokeCount, IterationMode.Workload, false, unrollFactor, forceSpecific); - - internal IReadOnlyList Run(long invokeCount, IterationMode iterationMode, bool runAuto, int unrollFactor, bool forceSpecific = false) - => (runAuto || iterationCount == null) && !forceSpecific - ? RunAuto(invokeCount, iterationMode, unrollFactor) - : RunSpecific(invokeCount, iterationMode, iterationCount ?? DefaultWorkloadCount, unrollFactor); - - private List RunAuto(long invokeCount, IterationMode iterationMode, int unrollFactor) - { - var measurements = new List(maxIterationCount); - var measurementsForStatistics = new List(maxIterationCount); - - int iterationCounter = 0; - bool isOverhead = iterationMode == IterationMode.Overhead; - double effectiveMaxRelativeError = isOverhead ? MaxOverheadRelativeError : maxRelativeError; - while (true) - { - iterationCounter++; - var measurement = RunIteration(iterationMode, IterationStage.Actual, iterationCounter, invokeCount, unrollFactor); - measurements.Add(measurement); - measurementsForStatistics.Add(measurement); - - var statistics = MeasurementsStatistics.Calculate(measurementsForStatistics, outlierMode); - double actualError = statistics.LegacyConfidenceInterval.Margin; - - double maxError1 = effectiveMaxRelativeError * statistics.Mean; - double maxError2 = maxAbsoluteError?.Nanoseconds ?? double.MaxValue; - double maxError = Math.Min(maxError1, maxError2); - - if (iterationCounter >= minIterationCount && actualError < maxError) - break; - - if (iterationCounter >= maxIterationCount || isOverhead && iterationCounter >= MaxOverheadIterationCount) - break; - } - WriteLine(); - - return measurements; - } - - private List RunSpecific(long invokeCount, IterationMode iterationMode, int iterationCount, int unrollFactor) - { - var measurements = new List(iterationCount); - - for (int i = 0; i < iterationCount; i++) - measurements.Add(RunIteration(iterationMode, IterationStage.Actual, i + 1, invokeCount, unrollFactor)); - - WriteLine(); - - return measurements; + + internal static EngineActualStage GetOverhead(IEngine engine) + => new EngineActualStageAuto(engine.TargetJob, engine.Resolver, IterationMode.Overhead); + + internal static EngineActualStage GetWorkload(IEngine engine, RunStrategy strategy) + { + var targetJob = engine.TargetJob; + int? iterationCount = targetJob.ResolveValueAsNullable(RunMode.IterationCountCharacteristic); + const int DefaultWorkloadCount = 10; + return iterationCount == null && strategy != RunStrategy.Monitoring + ? new EngineActualStageAuto(targetJob, engine.Resolver, IterationMode.Workload) + : new EngineActualStageSpecific(iterationCount ?? DefaultWorkloadCount, IterationMode.Workload); } - - internal IEngineStageEvaluator GetOverheadEvaluator() - => new AutoEvaluator(this, true); - - internal IEngineStageEvaluator GetWorkloadEvaluator(bool forceSpecific) - => iterationCount == null && !forceSpecific - ? new AutoEvaluator(this, false) - : new SpecificEvaluator(this); - - private sealed class AutoEvaluator(EngineActualStage stage, bool isOverhead) : IEngineStageEvaluator - { - public int MaxIterationCount => stage.maxIterationCount; + } - private readonly List _measurementsForStatistics = new (stage.maxIterationCount); - private int _iterationCounter = 0; - - public bool EvaluateShouldStop(List measurements, ref long invokeCount) + internal sealed class EngineActualStageAuto(Job targetJob, IResolver resolver, IterationMode iterationMode) : EngineActualStage(iterationMode) + { + private readonly double maxRelativeError = targetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, resolver); + private readonly TimeInterval? maxAbsoluteError = targetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic); + private readonly OutlierMode outlierMode = targetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, resolver); + private readonly int minIterationCount = targetJob.ResolveValue(RunMode.MinIterationCountCharacteristic, resolver); + private readonly int maxIterationCount = targetJob.ResolveValue(RunMode.MaxIterationCountCharacteristic, resolver); + private int _iterationCounter = 0; + + internal override List GetMeasurementList() => new (maxIterationCount); + + internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) + { + if (measurements.Count == 0) { - if (measurements.Count == 0) - { - return false; - } + return true; + } + + const double MaxOverheadRelativeError = 0.05; + bool isOverhead = Mode == IterationMode.Overhead; + double effectiveMaxRelativeError = isOverhead ? MaxOverheadRelativeError : maxRelativeError; + _iterationCounter++; - double effectiveMaxRelativeError = isOverhead ? MaxOverheadRelativeError : stage.maxRelativeError; - _iterationCounter++; - var measurement = measurements[measurements.Count - 1]; - _measurementsForStatistics.Add(measurement); + var statistics = MeasurementsStatistics.Calculate(measurements, outlierMode); + double actualError = statistics.LegacyConfidenceInterval.Margin; - var statistics = MeasurementsStatistics.Calculate(_measurementsForStatistics, stage.outlierMode); - double actualError = statistics.LegacyConfidenceInterval.Margin; + double maxError1 = effectiveMaxRelativeError * statistics.Mean; + double maxError2 = maxAbsoluteError?.Nanoseconds ?? double.MaxValue; + double maxError = Math.Min(maxError1, maxError2); - double maxError1 = effectiveMaxRelativeError * statistics.Mean; - double maxError2 = stage.maxAbsoluteError?.Nanoseconds ?? double.MaxValue; - double maxError = Math.Min(maxError1, maxError2); + if (_iterationCounter >= minIterationCount && actualError < maxError) + { + return false; + } - if (_iterationCounter >= stage.minIterationCount && actualError < maxError) - { - return true; - } + if (_iterationCounter >= maxIterationCount || isOverhead && _iterationCounter >= MaxOverheadIterationCount) + { + return false; + } - if (_iterationCounter >= stage.maxIterationCount || isOverhead && _iterationCounter >= MaxOverheadIterationCount) - { - return true; - } + return true; + } + } - return false; - } - } - - private sealed class SpecificEvaluator(EngineActualStage stage) : IEngineStageEvaluator - { - public int MaxIterationCount => stage.iterationCount ?? DefaultWorkloadCount; - - private int _iterationCount = 0; - - public bool EvaluateShouldStop(List measurements, ref long invokeCount) - => ++_iterationCount > MaxIterationCount; - } + internal sealed class EngineActualStageSpecific(int maxIterationCount, IterationMode iterationMode) : EngineActualStage(iterationMode) + { + private int iterationCount = 0; + + internal override List GetMeasurementList() => new (maxIterationCount); + + internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) + => ++iterationCount <= maxIterationCount; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/EnginePilotStage.cs b/src/BenchmarkDotNet/Engines/EnginePilotStage.cs index 03d079443e..afdbf379ef 100644 --- a/src/BenchmarkDotNet/Engines/EnginePilotStage.cs +++ b/src/BenchmarkDotNet/Engines/EnginePilotStage.cs @@ -1,219 +1,113 @@ using System; using System.Collections.Generic; +using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Reports; using Perfolizer.Horology; namespace BenchmarkDotNet.Engines -{ - // TODO: use clockResolution - internal class EnginePilotStage : EngineStage +{ + // TODO: use clockResolution + internal abstract class EnginePilotStage(Job targetJob, IResolver resolver) : EngineStage(IterationStage.Pilot, IterationMode.Workload) { - public readonly struct PilotStageResult - { - public long PerfectInvocationCount { get; } - public IReadOnlyList Measurements { get; } - - public PilotStageResult(long perfectInvocationCount, List measurements) - { - PerfectInvocationCount = perfectInvocationCount; - Measurements = measurements; - } - - public PilotStageResult(long perfectInvocationCount) - { - PerfectInvocationCount = perfectInvocationCount; - Measurements = Array.Empty(); - } - } - internal const long MaxInvokeCount = (long.MaxValue / 2 + 1) / 2; - private readonly int unrollFactor; - private readonly TimeInterval minIterationTime; - private readonly int minInvokeCount; - private readonly double maxRelativeError; - private readonly TimeInterval? maxAbsoluteError; - private readonly double targetIterationTime; - private readonly double resolution; - - public EnginePilotStage(IEngine engine) : base(engine) - { - unrollFactor = engine.TargetJob.ResolveValue(RunMode.UnrollFactorCharacteristic, engine.Resolver); - minIterationTime = engine.TargetJob.ResolveValue(AccuracyMode.MinIterationTimeCharacteristic, engine.Resolver); - minInvokeCount = engine.TargetJob.ResolveValue(AccuracyMode.MinInvokeCountCharacteristic, engine.Resolver); - maxRelativeError = engine.TargetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, engine.Resolver); - maxAbsoluteError = engine.TargetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic); - targetIterationTime = engine.TargetJob.ResolveValue(RunMode.IterationTimeCharacteristic, engine.Resolver).ToNanoseconds(); - resolution = engine.TargetJob.ResolveValue(InfrastructureMode.ClockCharacteristic, engine.Resolver).GetResolution().Nanoseconds; - } - - /// Perfect invocation count - public PilotStageResult Run() - { - // If InvocationCount is specified, pilot stage should be skipped - if (TargetJob.HasValue(RunMode.InvocationCountCharacteristic)) - return new PilotStageResult(TargetJob.Run.InvocationCount); - - // Here we want to guess "perfect" amount of invocation - return TargetJob.HasValue(RunMode.IterationTimeCharacteristic) - ? RunSpecific() - : RunAuto(); - } - - /// - /// A case where we don't have specific iteration time. - /// - private PilotStageResult RunAuto() - { - long invokeCount = Autocorrect(minInvokeCount); - var measurements = new List(); - - int iterationCounter = 0; - while (true) - { - iterationCounter++; - var measurement = RunIteration(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, unrollFactor); - measurements.Add(measurement); - double iterationTime = measurement.Nanoseconds; - double operationError = 2.0 * resolution / invokeCount; // An operation error which has arisen due to the Chronometer precision - - // Max acceptable operation error - double operationMaxError1 = iterationTime / invokeCount * maxRelativeError; - double operationMaxError2 = maxAbsoluteError?.Nanoseconds ?? double.MaxValue; - double operationMaxError = Math.Min(operationMaxError1, operationMaxError2); - - bool isFinished = operationError < operationMaxError && iterationTime >= minIterationTime.Nanoseconds; - if (isFinished) - break; - if (invokeCount >= MaxInvokeCount) - break; - - if (unrollFactor == 1 && invokeCount < EnvironmentResolver.DefaultUnrollFactorForThroughput) - invokeCount += 1; - else - invokeCount *= 2; - } - WriteLine(); - - return new PilotStageResult(invokeCount, measurements); - } - - /// - /// A case where we have specific iteration time. - /// - private PilotStageResult RunSpecific() - { - long invokeCount = Autocorrect(Engine.MinInvokeCount); - var measurements = new List(); - - int iterationCounter = 0; - - int downCount = 0; // Amount of iterations where newInvokeCount < invokeCount - while (true) - { - iterationCounter++; - var measurement = RunIteration(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, unrollFactor); - measurements.Add(measurement); - double actualIterationTime = measurement.Nanoseconds; - long newInvokeCount = Autocorrect(Math.Max(minInvokeCount, (long)Math.Round(invokeCount * targetIterationTime / actualIterationTime))); - - if (newInvokeCount < invokeCount) - downCount++; - - if (Math.Abs(newInvokeCount - invokeCount) <= 1 || downCount >= 3) - break; - - invokeCount = newInvokeCount; - } - WriteLine(); - - return new PilotStageResult(invokeCount, measurements); - } - - private long Autocorrect(long count) => (count + unrollFactor - 1) / unrollFactor * unrollFactor; - - internal IEngineStageEvaluator GetEvaluator() - { - // If InvocationCount is specified, pilot stage should be skipped - return TargetJob.HasValue(RunMode.InvocationCountCharacteristic) ? null - // Here we want to guess "perfect" amount of invocation - : TargetJob.HasValue(RunMode.IterationTimeCharacteristic) ? new SpecificEvaluator(this) - : new AutoEvaluator(this); - } - - private sealed class AutoEvaluator(EnginePilotStage stage) : IEngineStageEvaluator + protected readonly int unrollFactor = targetJob.ResolveValue(RunMode.UnrollFactorCharacteristic, resolver); + protected readonly int minInvokeCount = targetJob.ResolveValue(AccuracyMode.MinInvokeCountCharacteristic, resolver); + + protected long Autocorrect(long count) => (count + unrollFactor - 1) / unrollFactor * unrollFactor; + + internal static EnginePilotStage GetStage(IEngine engine) { - public int MaxIterationCount => 0; + var targetJob = engine.TargetJob; + // If InvocationCount is specified, pilot stage should be skipped + return targetJob.HasValue(RunMode.InvocationCountCharacteristic) ? null + // Here we want to guess "perfect" amount of invocation + : targetJob.HasValue(RunMode.IterationTimeCharacteristic) ? new EnginePilotStageSpecific(targetJob, engine.Resolver) + : new EnginePilotStageAuto(targetJob, engine.Resolver); + } + } - public bool EvaluateShouldStop(List measurements, ref long invokeCount) + internal sealed class EnginePilotStageAuto(Job targetJob, IResolver resolver) : EnginePilotStage(targetJob, resolver) + { + private readonly TimeInterval minIterationTime = targetJob.ResolveValue(AccuracyMode.MinIterationTimeCharacteristic, resolver); + private readonly double maxRelativeError = targetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, resolver); + private readonly TimeInterval? maxAbsoluteError = targetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic); + private readonly double resolution = targetJob.ResolveValue(InfrastructureMode.ClockCharacteristic, resolver).GetResolution().Nanoseconds; + + internal override List GetMeasurementList() => []; + + internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) + { + if (measurements.Count == 0) { - if (measurements.Count == 0) - { - invokeCount = stage.Autocorrect(stage.minInvokeCount); - return false; - } - - var measurement = measurements[measurements.Count - 1]; - double iterationTime = measurement.Nanoseconds; - double operationError = 2.0 * stage.resolution / invokeCount; // An operation error which has arisen due to the Chronometer precision - - // Max acceptable operation error - double operationMaxError1 = iterationTime / invokeCount * stage.maxRelativeError; - double operationMaxError2 = stage.maxAbsoluteError?.Nanoseconds ?? double.MaxValue; - double operationMaxError = Math.Min(operationMaxError1, operationMaxError2); - - bool isFinished = operationError < operationMaxError && iterationTime >= stage.minIterationTime.Nanoseconds; - if (isFinished || invokeCount >= MaxInvokeCount) - { - return true; - } - - if (stage.unrollFactor == 1 && invokeCount < EnvironmentResolver.DefaultUnrollFactorForThroughput) - { - ++invokeCount; - } - else - { - invokeCount *= 2; - } + invokeCount = Autocorrect(minInvokeCount); + return true; + } + var measurement = measurements[measurements.Count - 1]; + double iterationTime = measurement.Nanoseconds; + double operationError = 2.0 * resolution / invokeCount; // An operation error which has arisen due to the Chronometer precision + + // Max acceptable operation error + double operationMaxError1 = iterationTime / invokeCount * maxRelativeError; + double operationMaxError2 = maxAbsoluteError?.Nanoseconds ?? double.MaxValue; + double operationMaxError = Math.Min(operationMaxError1, operationMaxError2); + + bool isFinished = operationError < operationMaxError && iterationTime >= minIterationTime.Nanoseconds; + if (isFinished || invokeCount >= MaxInvokeCount) + { return false; } - } - - private sealed class SpecificEvaluator(EnginePilotStage stage) : IEngineStageEvaluator + + if (unrollFactor == 1 && invokeCount < EnvironmentResolver.DefaultUnrollFactorForThroughput) + { + ++invokeCount; + } + else + { + invokeCount *= 2; + } + + return true; + } + } + + internal sealed class EnginePilotStageSpecific(Job targetJob, IResolver resolver) : EnginePilotStage(targetJob, resolver) + { + private const int MinInvokeCount = 4; + + private readonly double targetIterationTime = targetJob.ResolveValue(RunMode.IterationTimeCharacteristic, resolver).ToNanoseconds(); + + private int _downCount = 0; // Amount of iterations where newInvokeCount < invokeCount + + internal override List GetMeasurementList() => []; + + internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) { - public int MaxIterationCount => 0; + if (measurements.Count == 0) + { + invokeCount = Autocorrect(MinInvokeCount); + return true; + } + + var measurement = measurements[measurements.Count - 1]; + double actualIterationTime = measurement.Nanoseconds; + long newInvokeCount = Autocorrect(Math.Max(minInvokeCount, (long) Math.Round(invokeCount * targetIterationTime / actualIterationTime))); - private int _downCount = 0; // Amount of iterations where newInvokeCount < invokeCount + if (newInvokeCount < invokeCount) + { + _downCount++; + } - public bool EvaluateShouldStop(List measurements, ref long invokeCount) + if (Math.Abs(newInvokeCount - invokeCount) <= 1 || _downCount >= 3) { - if (measurements.Count == 0) - { - invokeCount = stage.Autocorrect(Engine.MinInvokeCount); - return false; - } - - var measurement = measurements[measurements.Count - 1]; - double actualIterationTime = measurement.Nanoseconds; - long newInvokeCount = stage.Autocorrect(Math.Max(stage.minInvokeCount, (long) Math.Round(invokeCount * stage.targetIterationTime / actualIterationTime))); - - if (newInvokeCount < invokeCount) - { - _downCount++; - } - - if (Math.Abs(newInvokeCount - invokeCount) <= 1 || _downCount >= 3) - { - return true; - } - - invokeCount = newInvokeCount; return false; } - } + + invokeCount = newInvokeCount; + return true; + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/EngineStage.cs b/src/BenchmarkDotNet/Engines/EngineStage.cs index 2c3b825394..82b6bc5fb3 100644 --- a/src/BenchmarkDotNet/Engines/EngineStage.cs +++ b/src/BenchmarkDotNet/Engines/EngineStage.cs @@ -1,48 +1,43 @@ -using System; -using System.Collections.Generic; -using BenchmarkDotNet.Jobs; +using System.Collections.Generic; +using System.Runtime.CompilerServices; using BenchmarkDotNet.Reports; namespace BenchmarkDotNet.Engines { - public class EngineStage + internal abstract class EngineStage(IterationStage stage, IterationMode mode) { - private readonly IEngine engine; + internal readonly IterationStage Stage = stage; + internal readonly IterationMode Mode = mode; - protected EngineStage(IEngine engine) => this.engine = engine; + internal abstract List GetMeasurementList(); + internal abstract bool GetShouldRunIteration(List measurements, ref long invokeCount); - protected Job TargetJob => engine.TargetJob; - - protected Measurement RunIteration(IterationMode mode, IterationStage stage, int index, long invokeCount, int unrollFactor) - { - if (invokeCount % unrollFactor != 0) - throw new ArgumentOutOfRangeException($"InvokeCount({invokeCount}) should be a multiple of UnrollFactor({unrollFactor})."); - return engine.RunIteration(new IterationData(mode, stage, index, invokeCount, unrollFactor)); - } - - internal List Run(IStoppingCriteria criteria, long invokeCount, IterationMode mode, IterationStage stage, int unrollFactor) + [MethodImpl(MethodImplOptions.NoInlining)] + internal static IEnumerable EnumerateStages(IEngine engine, RunStrategy strategy, bool evaluateOverhead) { - var measurements = new List(criteria.MaxIterationCount); - if (criteria.Evaluate(measurements).IsFinished) - { - WriteLine(); - return measurements; - } + // It might be possible to add the jitting stage to this, but it's done in EngineFactory.CreateReadyToRun for now. - int iterationCounter = 0; - while (true) + if (strategy != RunStrategy.ColdStart) { - iterationCounter++; - measurements.Add(RunIteration(mode, stage, iterationCounter, invokeCount, unrollFactor)); - if (criteria.Evaluate(measurements).IsFinished) - break; + if (strategy != RunStrategy.Monitoring) + { + var pilotStage = EnginePilotStage.GetStage(engine); + if (pilotStage != null) + { + yield return pilotStage; + } + + if (evaluateOverhead) + { + yield return EngineWarmupStage.GetOverhead(); + yield return EngineActualStage.GetOverhead(engine); + } + } + + yield return EngineWarmupStage.GetWorkload(engine, strategy); } - WriteLine(); - - return measurements; + yield return EngineActualStage.GetWorkload(engine, strategy); } - - protected void WriteLine() => engine.WriteLine(); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs b/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs index 12c5e964ba..a82ca61a91 100644 --- a/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs +++ b/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs @@ -1,38 +1,78 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Reports; namespace BenchmarkDotNet.Engines { - internal class EngineWarmupStage : EngineStage + internal abstract class EngineWarmupStage(IterationMode iterationMode) : EngineStage(IterationStage.Warmup, iterationMode) { - private readonly IEngine engine; + private const int MinOverheadIterationCount = 4; + internal const int MaxOverheadIterationCount = 10; - public EngineWarmupStage(IEngine engine) : base(engine) => this.engine = engine; + internal static EngineWarmupStage GetOverhead() + => new EngineWarmupStageAuto(IterationMode.Overhead, MinOverheadIterationCount, MaxOverheadIterationCount); - public IReadOnlyList RunOverhead(long invokeCount, int unrollFactor) - => Run(invokeCount, IterationMode.Overhead, unrollFactor, RunStrategy.Throughput); - - public IReadOnlyList RunWorkload(long invokeCount, int unrollFactor, RunStrategy runStrategy) - => Run(invokeCount, IterationMode.Workload, unrollFactor, runStrategy); - - internal IReadOnlyList Run(long invokeCount, IterationMode iterationMode, int unrollFactor, RunStrategy runStrategy) + internal static EngineWarmupStage GetWorkload(IEngine engine, RunStrategy runStrategy) { - var criteria = DefaultStoppingCriteriaFactory.Instance.CreateWarmup(engine.TargetJob, engine.Resolver, iterationMode, runStrategy); - return Run(criteria, invokeCount, iterationMode, IterationStage.Warmup, unrollFactor); + var job = engine.TargetJob; + var count = job.ResolveValueAsNullable(RunMode.WarmupCountCharacteristic); + if (count.HasValue && count.Value != EngineResolver.ForceAutoWarmup || runStrategy == RunStrategy.Monitoring) + { + return new EngineWarmupStageSpecific(count ?? 0, IterationMode.Workload); + } + + int minIterationCount = job.ResolveValue(RunMode.MinWarmupIterationCountCharacteristic, engine.Resolver); + int maxIterationCount = job.ResolveValue(RunMode.MaxWarmupIterationCountCharacteristic, engine.Resolver); + return new EngineWarmupStageAuto(IterationMode.Overhead, minIterationCount, maxIterationCount); } + } - internal IEngineStageEvaluator GetOverheadEvaluator() - => new Evaluator(DefaultStoppingCriteriaFactory.Instance.CreateWarmup(engine.TargetJob, engine.Resolver, IterationMode.Overhead, RunStrategy.Throughput)); + internal sealed class EngineWarmupStageAuto(IterationMode iterationMode, int minIterationCount, int maxIterationCount) : EngineWarmupStage(iterationMode) + { + private const int MinFluctuationCount = 4; + + private readonly int minIterationCount = minIterationCount; + private readonly int maxIterationCount = maxIterationCount; - internal IEngineStageEvaluator GetWorkloadEvaluator(RunStrategy runStrategy) - => new Evaluator(DefaultStoppingCriteriaFactory.Instance.CreateWarmup(engine.TargetJob, engine.Resolver, IterationMode.Workload, runStrategy)); + internal override List GetMeasurementList() => new (maxIterationCount); - private sealed class Evaluator(IStoppingCriteria stoppingCriteria) : IEngineStageEvaluator + internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) { - public int MaxIterationCount => stoppingCriteria.MaxIterationCount; + int n = measurements.Count; + + if (n >= maxIterationCount) + { + return false; + } + if (n < minIterationCount) + { + return true; + } - public bool EvaluateShouldStop(List measurements, ref long invokeCount) - => stoppingCriteria.Evaluate(measurements).IsFinished; + int direction = -1; // The default "pre-state" is "decrease mode" + int fluctuationCount = 0; + for (int i = 1; i < n; i++) + { + int nextDirection = Math.Sign(measurements[i].Nanoseconds - measurements[i - 1].Nanoseconds); + if (nextDirection != direction || nextDirection == 0) + { + direction = nextDirection; + fluctuationCount++; + } + } + + return fluctuationCount < MinFluctuationCount; } } + + internal sealed class EngineWarmupStageSpecific(int maxIterationCount, IterationMode iterationMode) : EngineWarmupStage(iterationMode) + { + private int iterationCount = 0; + + internal override List GetMeasurementList() => new (maxIterationCount); + + internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) + => ++iterationCount <= maxIterationCount; + } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/IEngineStageEvaluator.cs b/src/BenchmarkDotNet/Engines/IEngineStageEvaluator.cs deleted file mode 100644 index b2eaa5316f..0000000000 --- a/src/BenchmarkDotNet/Engines/IEngineStageEvaluator.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System.Collections.Generic; -using BenchmarkDotNet.Reports; - -namespace BenchmarkDotNet.Engines -{ - internal interface IEngineStageEvaluator - { - bool EvaluateShouldStop(List measurements, ref long invokeCount); - int MaxIterationCount { get; } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/StoppingCriteria/AutoWarmupStoppingCriteria.cs b/src/BenchmarkDotNet/Engines/StoppingCriteria/AutoWarmupStoppingCriteria.cs deleted file mode 100644 index e2282c7baa..0000000000 --- a/src/BenchmarkDotNet/Engines/StoppingCriteria/AutoWarmupStoppingCriteria.cs +++ /dev/null @@ -1,97 +0,0 @@ -using System; -using System.Collections.Generic; -using BenchmarkDotNet.Reports; - -namespace BenchmarkDotNet.Engines -{ - /// - /// Automatically choose the best number of iterations during the warmup stage. - /// - public class AutoWarmupStoppingCriteria : StoppingCriteriaBase - { - private const int DefaultMinFluctuationCount = 4; - - private readonly int minIterationCount; - private readonly int maxIterationCount; - private readonly int minFluctuationCount; - - private readonly string maxIterationMessage; - private readonly string minFluctuationMessage; - - /// - /// The idea of the implementation is simple: if measurements monotonously decrease or increase, the steady state is not achieved; - /// we should continue the warmup stage. The evaluation method counts "fluctuations" - /// (3 consecutive measurements A,B,C where (A>B AND B<C) OR (A<B AND B>C)) until the required amount of flotations is observed. - /// - /// - /// The minimum number of iterations. - /// We are always going to do at least iterations regardless of fluctuations. - /// - /// - /// The maximum number of iterations. - /// We are always going to do at most iterations regardless of fluctuations. - /// - /// - /// The required number of fluctuations. - /// If the required number of fluctuations is achieved but the number of iterations less than , - /// we need more iterations. - /// If the required number of fluctuations is not achieved but the number of iterations equal to , - /// we should stop the iterations. - /// - public AutoWarmupStoppingCriteria(int minIterationCount, int maxIterationCount, int minFluctuationCount = DefaultMinFluctuationCount) - { - this.minIterationCount = minIterationCount; - this.maxIterationCount = maxIterationCount; - this.minFluctuationCount = minFluctuationCount; - - maxIterationMessage = $"The maximum amount of iteration ({maxIterationCount}) is achieved"; - minFluctuationMessage = $"The minimum amount of fluctuation ({minFluctuationCount}) and " + - $"the minimum amount of iterations ({minIterationCount}) are achieved"; - } - - public override StoppingResult Evaluate(IReadOnlyList measurements) - { - int n = measurements.Count; - - if (n >= maxIterationCount) - return StoppingResult.CreateFinished(maxIterationMessage); - if (n < minIterationCount) - return StoppingResult.NotFinished; - - int direction = -1; // The default "pre-state" is "decrease mode" - int fluctuationCount = 0; - for (int i = 1; i < n; i++) - { - int nextDirection = Math.Sign(measurements[i].Nanoseconds - measurements[i - 1].Nanoseconds); - if (nextDirection != direction || nextDirection == 0) - { - direction = nextDirection; - fluctuationCount++; - } - } - - return fluctuationCount >= minFluctuationCount - ? StoppingResult.CreateFinished(minFluctuationMessage) - : StoppingResult.NotFinished; - } - - protected override string GetTitle() => $"{nameof(AutoWarmupStoppingCriteria)}(" + - $"{nameof(minIterationCount)}={minIterationCount}, " + - $"{nameof(maxIterationCount)}={maxIterationCount}, " + - $"{nameof(minFluctuationCount)}={minFluctuationCount})"; - - protected override int GetMaxIterationCount() => maxIterationCount; - - protected override IEnumerable GetWarnings() - { - if (minIterationCount < 0) - yield return $"Min Iteration Count ({minIterationCount}) is negative"; - if (maxIterationCount < 0) - yield return $"Max Iteration Count ({maxIterationCount}) is negative"; - if (minFluctuationCount < 0) - yield return $"Min Fluctuation Count ({minFluctuationCount}) is negative"; - if (minIterationCount > maxIterationCount) - yield return $"Min Iteration Count ({minFluctuationCount}) is greater than Max Iteration Count ({maxIterationCount})"; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/StoppingCriteria/DefaultStoppingCriteriaFactory.cs b/src/BenchmarkDotNet/Engines/StoppingCriteria/DefaultStoppingCriteriaFactory.cs deleted file mode 100644 index 356fc7362e..0000000000 --- a/src/BenchmarkDotNet/Engines/StoppingCriteria/DefaultStoppingCriteriaFactory.cs +++ /dev/null @@ -1,43 +0,0 @@ -using System; -using BenchmarkDotNet.Characteristics; -using BenchmarkDotNet.Jobs; - -namespace BenchmarkDotNet.Engines -{ - public class DefaultStoppingCriteriaFactory - { - public static readonly DefaultStoppingCriteriaFactory Instance = new DefaultStoppingCriteriaFactory(); - - private const int MinOverheadIterationCount = 4; - internal const int MaxOverheadIterationCount = 10; - - public virtual IStoppingCriteria CreateWarmupWorkload(Job job, IResolver resolver, RunStrategy runStrategy) - { - var count = job.ResolveValueAsNullable(RunMode.WarmupCountCharacteristic); - if (count.HasValue && count.Value != EngineResolver.ForceAutoWarmup || runStrategy == RunStrategy.Monitoring) - return new FixedStoppingCriteria(count ?? 0); - - int minIterationCount = job.ResolveValue(RunMode.MinWarmupIterationCountCharacteristic, resolver); - int maxIterationCount = job.ResolveValue(RunMode.MaxWarmupIterationCountCharacteristic, resolver); - return new AutoWarmupStoppingCriteria(minIterationCount, maxIterationCount); - } - - public virtual IStoppingCriteria CreateWarmupOverhead() - { - return new AutoWarmupStoppingCriteria(MinOverheadIterationCount, MaxOverheadIterationCount); - } - - public virtual IStoppingCriteria CreateWarmup(Job job, IResolver resolver, IterationMode mode, RunStrategy runStrategy) - { - switch (mode) - { - case IterationMode.Overhead: - return CreateWarmupOverhead(); - case IterationMode.Workload: - return CreateWarmupWorkload(job, resolver, runStrategy); - default: - throw new ArgumentOutOfRangeException(nameof(mode), mode, null); - } - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/StoppingCriteria/FixedStoppingCriteria.cs b/src/BenchmarkDotNet/Engines/StoppingCriteria/FixedStoppingCriteria.cs deleted file mode 100644 index 847bb41a59..0000000000 --- a/src/BenchmarkDotNet/Engines/StoppingCriteria/FixedStoppingCriteria.cs +++ /dev/null @@ -1,37 +0,0 @@ -using System.Collections.Generic; -using BenchmarkDotNet.Reports; - -namespace BenchmarkDotNet.Engines -{ - /// - /// Stopping criteria which require a specific amount of iterations. - /// - public class FixedStoppingCriteria : StoppingCriteriaBase - { - private readonly int iterationCount; - - private readonly string message; - - public FixedStoppingCriteria(int iterationCount) - { - this.iterationCount = iterationCount; - - message = $"The required amount of iteration ({iterationCount}) is achieved"; - } - - public override StoppingResult Evaluate(IReadOnlyList measurements) - => measurements.Count >= MaxIterationCount - ? StoppingResult.CreateFinished(message) - : StoppingResult.NotFinished; - - protected override string GetTitle() => $"{nameof(FixedStoppingCriteria)}({nameof(iterationCount)}={iterationCount})"; - - protected override int GetMaxIterationCount() => iterationCount; - - protected override IEnumerable GetWarnings() - { - if (iterationCount < 0) - yield return $"Iteration count ({iterationCount}) is negative"; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/StoppingCriteria/IStoppingCriteria.cs b/src/BenchmarkDotNet/Engines/StoppingCriteria/IStoppingCriteria.cs deleted file mode 100644 index 53550ef919..0000000000 --- a/src/BenchmarkDotNet/Engines/StoppingCriteria/IStoppingCriteria.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System.Collections.Generic; -using BenchmarkDotNet.Reports; - -namespace BenchmarkDotNet.Engines -{ - /// - /// A stopping criteria checks when it's time to terminate iteration in the current stage. - /// - public interface IStoppingCriteria - { - /// - /// Checks do we have enough iterations - /// - StoppingResult Evaluate(IReadOnlyList measurements); - - /// - /// Title which can be used in logs and diagnostics methods - /// - string Title { get; } - - /// - /// The maximum possible count of iterations. - /// Engine needs this value for setting the maximum capacity of the returned list of measurements. - /// The correct capacity helps to avoid infrastructure allocations during benchmarking. - /// - int MaxIterationCount { get; } - - /// - /// An array of user-friendly warnings which notify about incorrect parameters. - /// - IReadOnlyList Warnings { get; } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/StoppingCriteria/StoppingCriteriaBase.cs b/src/BenchmarkDotNet/Engines/StoppingCriteria/StoppingCriteriaBase.cs deleted file mode 100644 index 5956e37760..0000000000 --- a/src/BenchmarkDotNet/Engines/StoppingCriteria/StoppingCriteriaBase.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using BenchmarkDotNet.Reports; - -namespace BenchmarkDotNet.Engines -{ - public abstract class StoppingCriteriaBase : IStoppingCriteria - { - private readonly Lazy lazyTitle; - private readonly Lazy lazyMaxIterationCount; - private readonly Lazy lazyWarnings; - - public string Title => lazyTitle.Value; - public int MaxIterationCount => lazyMaxIterationCount.Value; - public IReadOnlyList Warnings => lazyWarnings.Value; - - protected StoppingCriteriaBase() - { - lazyTitle = new Lazy(GetTitle); - lazyMaxIterationCount = new Lazy(GetMaxIterationCount); - lazyWarnings = new Lazy(() => GetWarnings().ToArray()); - } - - public abstract StoppingResult Evaluate(IReadOnlyList measurements); - - protected abstract string GetTitle(); - - protected abstract int GetMaxIterationCount(); - - protected abstract IEnumerable GetWarnings(); - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/StoppingCriteria/StoppingResult.cs b/src/BenchmarkDotNet/Engines/StoppingCriteria/StoppingResult.cs deleted file mode 100644 index a51287c4b4..0000000000 --- a/src/BenchmarkDotNet/Engines/StoppingCriteria/StoppingResult.cs +++ /dev/null @@ -1,18 +0,0 @@ -namespace BenchmarkDotNet.Engines -{ - public struct StoppingResult - { - public readonly bool IsFinished; - - public readonly string? Message; - - private StoppingResult(bool isFinished, string? message) - { - IsFinished = isFinished; - Message = message; - } - - public static readonly StoppingResult NotFinished = new StoppingResult(false, null); - public static StoppingResult CreateFinished(string message) => new StoppingResult(true, message); - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineGeneralStageTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineActualStageTests.cs similarity index 85% rename from tests/BenchmarkDotNet.Tests/Engine/EngineGeneralStageTests.cs rename to tests/BenchmarkDotNet.Tests/Engine/EngineActualStageTests.cs index 5811a0f68c..97078eb953 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineGeneralStageTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineActualStageTests.cs @@ -36,17 +36,14 @@ private void AutoTest(Func measure, int min, int ma if (max == -1) max = min; var job = Job.Default; - var stage = CreateStage(job, measure); - var measurements = stage.Run(1, iterationMode, true, 1); + var engine = new MockEngine(output, job, measure); + var stage = iterationMode == IterationMode.Overhead + ? EngineActualStage.GetOverhead(engine) + : EngineActualStage.GetWorkload(engine, RunStrategy.Throughput); + var (_, measurements) = engine.Run(stage); int count = measurements.Count; output.WriteLine($"MeasurementCount = {count} (Min= {min}, Max = {max})"); Assert.InRange(count, min, max); } - - private EngineActualStage CreateStage(Job job, Func measure) - { - var engine = new MockEngine(output, job, measure); - return new EngineActualStage(engine); - } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs index d5b7b8c65c..fd3f4887ff 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs @@ -1,6 +1,8 @@ using System; +using System.Collections.Generic; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Reports; using BenchmarkDotNet.Tests.Mocks; using Perfolizer.Horology; using Xunit; @@ -46,8 +48,8 @@ private void AutoTest(Frequency clockFrequency, TimeInterval operationTime, doub Infrastructure = { Clock = new MockClock(clockFrequency) }, Accuracy = { MaxRelativeError = maxRelativeError } }.Freeze(); - var stage = CreateStage(job, data => data.InvokeCount * operationTime); - long invokeCount = stage.Run().PerfectInvocationCount; + var engine = new MockEngine(output, job, data => data.InvokeCount * operationTime); + var (invokeCount, _) = engine.Run(EnginePilotStage.GetStage(engine)); output.WriteLine($"InvokeCount = {invokeCount} (Min= {minInvokeCount}, Max = {MaxPossibleInvokeCount})"); Assert.InRange(invokeCount, minInvokeCount, MaxPossibleInvokeCount); } @@ -59,16 +61,10 @@ private void SpecificTest(TimeInterval iterationTime, TimeInterval operationTime Infrastructure = { Clock = new MockClock(Frequency.MHz) }, Run = { IterationTime = iterationTime } }.Freeze(); - var stage = CreateStage(job, data => data.InvokeCount * operationTime); - long invokeCount = stage.Run().PerfectInvocationCount; + var engine = new MockEngine(output, job, data => data.InvokeCount * operationTime); + var (invokeCount, _) = engine.Run(EnginePilotStage.GetStage(engine)); output.WriteLine($"InvokeCount = {invokeCount} (Min= {minInvokeCount}, Max = {maxInvokeCount})"); Assert.InRange(invokeCount, minInvokeCount, maxInvokeCount); } - - private EnginePilotStage CreateStage(Job job, Func measure) - { - var engine = new MockEngine(output, job, measure); - return new EnginePilotStage(engine); - } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineWarmupStageTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineWarmupStageTests.cs index 1f966fbcaa..b9c465b87b 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineWarmupStageTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineWarmupStageTests.cs @@ -16,7 +16,7 @@ public class EngineWarmupStageTests { private const int MinIterationCount = EngineResolver.DefaultMinWarmupIterationCount; private const int MaxIterationCount = EngineResolver.DefaultMaxWarmupIterationCount; - private const int MaxOverheadIterationCount = DefaultStoppingCriteriaFactory.MaxOverheadIterationCount; + private const int MaxOverheadIterationCount = EngineWarmupStage.MaxOverheadIterationCount; private readonly ITestOutputHelper output; @@ -85,17 +85,14 @@ private void AutoTest(Func measure, int min, int ma { if (max == -1) max = min; - var stage = CreateStage(job ?? Job.Default, measure); - var measurements = stage.Run(1, mode, 1, RunStrategy.Throughput); + var engine = new MockEngine(output, job ?? Job.Default, measure); + var stage = mode == IterationMode.Overhead + ? EngineWarmupStage.GetOverhead() + : EngineWarmupStage.GetWorkload(engine, RunStrategy.Throughput); + var (_, measurements) = engine.Run(stage); int count = measurements.Count; output.WriteLine($"MeasurementCount = {count} (Min= {min}, Max = {max})"); Assert.InRange(count, min, max); } - - private EngineWarmupStage CreateStage(Job job, Func measure) - { - var engine = new MockEngine(output, job, measure); - return new EngineWarmupStage(engine); - } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/AutoWarmupStoppingCriteriaTests.cs b/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/AutoWarmupStoppingCriteriaTests.cs deleted file mode 100644 index 39eb94a225..0000000000 --- a/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/AutoWarmupStoppingCriteriaTests.cs +++ /dev/null @@ -1,80 +0,0 @@ -using System.Collections.Generic; -using BenchmarkDotNet.Engines; -using JetBrains.Annotations; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.Tests.Engine.StoppingCriteria -{ - public class AutoWarmupStoppingCriteriaTests : StoppingCriteriaTestsBase - { - public AutoWarmupStoppingCriteriaTests(ITestOutputHelper output) : base(output) { } - - [Theory] - [MemberData(nameof(EvaluateData))] - public void EvaluateTest(int minIterationCount, int maxIterationCount, int minFluctuationCount, double[] values, int expectedCount) - { - var criteria = new AutoWarmupStoppingCriteria(minIterationCount, maxIterationCount, minFluctuationCount); - ResolutionsAreCorrect(criteria, values, expectedCount); - } - - public static IEnumerable EvaluateData() - { - yield return new object[] { 0, 8, 4, new double[] { 0, 10, 0, 10, 0, 10, 0, 10, 0, 10 }, 5 }; - yield return new object[] { 0, 5, 1, new double[] { 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 }, 5 }; - yield return new object[] { 0, 5, 1, new double[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, 2 }; - yield return new object[] { 0, 600, 2, new double[] { 10, 9, 8, 7, 8, 9, 8, 7, 8, 9 }, 7 }; - yield return new object[] { 5, 8, 0, new double[] { 0, 10, 0, 10, 0, 10, 0, 10, 0, 10 }, 5 }; - } - - [Theory] - [InlineData(0, 0, 0, "AutoWarmupStoppingCriteria(minIterationCount=0, maxIterationCount=0, minFluctuationCount=0)")] - [InlineData(1, 22, 333, "AutoWarmupStoppingCriteria(minIterationCount=1, maxIterationCount=22, minFluctuationCount=333)")] - public void AutoWarmupTitleTest(int minIterationCount, int maxIterationCount, int minFluctuationCount, string expectedTitle) - { - var criteria = new AutoWarmupStoppingCriteria(minIterationCount, maxIterationCount, minFluctuationCount); - Assert.Equal(expectedTitle, criteria.Title); - } - - [Theory] - [MemberData(nameof(WarningsDataNames))] - public void WarningsTest(string dataName) - { - var (criteria, expectedWarnings) = WarningsData[dataName]; - Output.WriteLine("Criteria:" + criteria.Title); - Assert.Equal(expectedWarnings.Messages, criteria.Warnings); - } - - private static readonly IDictionary WarningsData = new Dictionary - { - { - "0/0/0", new WaringTestData( - new AutoWarmupStoppingCriteria(0, 0, 0), - Warnings.Empty) - }, - { - "5/4/0", new WaringTestData( - new AutoWarmupStoppingCriteria(5, 4, 0), - new Warnings("Min Iteration Count (0) is greater than Max Iteration Count (4)")) - }, - { - "-5/0/0", new WaringTestData( - new AutoWarmupStoppingCriteria(-5, 0, 0), - new Warnings("Min Iteration Count (-5) is negative")) - }, - { - "0/0/-5", new WaringTestData( - new AutoWarmupStoppingCriteria(0, 0, -5), - new Warnings("Min Fluctuation Count (-5) is negative")) - }, - { - "0/-5/0", new WaringTestData( - new AutoWarmupStoppingCriteria(0, -5, 0), - new Warnings("Max Iteration Count (-5) is negative", "Min Iteration Count (0) is greater than Max Iteration Count (-5)")) - } - }; - - [UsedImplicitly] - public static TheoryData WarningsDataNames = TheoryDataHelper.Create(WarningsData.Keys); - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/FixedStoppingCriteriaTests.cs b/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/FixedStoppingCriteriaTests.cs deleted file mode 100644 index 0a352d8c74..0000000000 --- a/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/FixedStoppingCriteriaTests.cs +++ /dev/null @@ -1,67 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using BenchmarkDotNet.Engines; -using JetBrains.Annotations; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.Tests.Engine.StoppingCriteria -{ - public class FixedStoppingCriteriaTests : StoppingCriteriaTestsBase - { - public FixedStoppingCriteriaTests(ITestOutputHelper output) : base(output) { } - - [Theory] - [InlineData(1)] - [InlineData(2)] - [InlineData(10)] - [InlineData(100)] - public void EvaluateTest(int expectedCount) - { - var values = Enumerable.Range(0, expectedCount * 2).Select(x => (double) x).ToArray(); - var criteria = new FixedStoppingCriteria(expectedCount); - ResolutionsAreCorrect(criteria, values, expectedCount); - } - - [Theory] - [InlineData(0, "FixedStoppingCriteria(iterationCount=0)")] - [InlineData(1, "FixedStoppingCriteria(iterationCount=1)")] - [InlineData(42, "FixedStoppingCriteria(iterationCount=42)")] - public void TitleTest(int iterationTitle, string expectedTitle) - { - var criteria = new FixedStoppingCriteria(iterationTitle); - Assert.Equal(expectedTitle, criteria.Title); - } - - [Theory] - [MemberData(nameof(WarningsDataNames))] - public void WarningsTest(string dataName) - { - var (criteria, expectedWarnings) = WarningsData[dataName]; - Output.WriteLine("Criteria:" + criteria.Title); - Assert.Equal(expectedWarnings.Messages, criteria.Warnings); - } - - private static readonly IDictionary WarningsData = new Dictionary - { - { - "0", new WaringTestData( - new FixedStoppingCriteria(0), - Warnings.Empty) - }, - { - "1", new WaringTestData( - new FixedStoppingCriteria(1), - Warnings.Empty) - }, - { - "-1", new WaringTestData( - new FixedStoppingCriteria(-1), - new Warnings("Iteration count (-1) is negative")) - } - }; - - [UsedImplicitly] - public static TheoryData WarningsDataNames = TheoryDataHelper.Create(WarningsData.Keys); - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/StoppingCriteriaTestsBase.cs b/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/StoppingCriteriaTestsBase.cs deleted file mode 100644 index 39b83da67a..0000000000 --- a/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/StoppingCriteriaTestsBase.cs +++ /dev/null @@ -1,73 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using BenchmarkDotNet.Engines; -using BenchmarkDotNet.Reports; -using JetBrains.Annotations; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.Tests.Engine.StoppingCriteria -{ - [Collection("StoppingCriteriaTests")] - public abstract class StoppingCriteriaTestsBase - { - protected readonly ITestOutputHelper Output; - - protected StoppingCriteriaTestsBase(ITestOutputHelper output) => Output = output; - - [AssertionMethod] - protected void ResolutionsAreCorrect(IStoppingCriteria criteria, double[] values, int expectedCount) - { - var measurements = Generate(values); - for (int iteration = 1; iteration <= measurements.Count; iteration++) - { - var subMeasurements = measurements.Take(iteration).ToList(); - var resolution = criteria.Evaluate(subMeasurements); - bool actualIsFinished = resolution.IsFinished; - bool expectedIsFinished = iteration >= expectedCount; - Output.WriteLine($"#{iteration} " + - $"Expected: {expectedIsFinished}, " + - $"Actual: {actualIsFinished}{(resolution.Message != null ? $" [{resolution.Message}]" : "")}, " + - $"Measurements: <{string.Join(",", values.Take(iteration))}>"); - Assert.Equal(expectedIsFinished, actualIsFinished); - } - } - - private static IReadOnlyList Generate(params double[] values) - { - var measurements = new List(values.Length); - for (int i = 1; i <= values.Length; i++) - measurements.Add(new Measurement(1, IterationMode.Unknown, IterationStage.Unknown, i, 1, values[i - 1])); - return measurements; - } - - public class Warnings - { - public static readonly Warnings Empty = new Warnings(); - - public string[] Messages { get; } - - public Warnings(params string[] messages) => Messages = messages; - - public override string ToString() => $"Messages: {Messages.Length}"; - } - - public class WaringTestData - { - private IStoppingCriteria Criteria { get; } - private Warnings ExpectedWarnings { get; } - - public WaringTestData(IStoppingCriteria criteria, Warnings expectedWarnings) - { - Criteria = criteria; - ExpectedWarnings = expectedWarnings; - } - - public void Deconstruct(out IStoppingCriteria criteria, out Warnings expectedWarnings) - { - criteria = Criteria; - expectedWarnings = ExpectedWarnings; - } - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs b/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs index 6b1fad64a6..7b17baa098 100644 --- a/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs +++ b/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Jobs; @@ -56,6 +57,19 @@ public Measurement RunIteration(IterationData data) public RunResults Run() => default; + internal (long invokeCount, List measurements) Run(EngineStage stage, long invokeCount = 0) + { + var measurements = stage.GetMeasurementList(); + int iterationIndex = 1; + while (stage.GetShouldRunIteration(measurements, ref invokeCount)) + { + var measurement = RunIteration(new IterationData(stage.Mode, stage.Stage, iterationIndex, invokeCount, 1)); + measurements.Add(measurement); + ++iterationIndex; + } + return (invokeCount, measurements); + } + public void WriteLine() => output.WriteLine(""); public void WriteLine(string line) => output.WriteLine(line); From 66260f9b3826678363e8606f6ac7c4178d65ae0b Mon Sep 17 00:00:00 2001 From: Tim Cassell Date: Sat, 3 May 2025 22:01:32 -0400 Subject: [PATCH 5/5] Fix measurementsForStatistics. --- .../Engines/EngineActualStage.cs | 35 +++++++++++++------ 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/src/BenchmarkDotNet/Engines/EngineActualStage.cs b/src/BenchmarkDotNet/Engines/EngineActualStage.cs index 2fbf9206ae..c736a07240 100644 --- a/src/BenchmarkDotNet/Engines/EngineActualStage.cs +++ b/src/BenchmarkDotNet/Engines/EngineActualStage.cs @@ -27,14 +27,25 @@ internal static EngineActualStage GetWorkload(IEngine engine, RunStrategy strate } } - internal sealed class EngineActualStageAuto(Job targetJob, IResolver resolver, IterationMode iterationMode) : EngineActualStage(iterationMode) + internal sealed class EngineActualStageAuto : EngineActualStage { - private readonly double maxRelativeError = targetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, resolver); - private readonly TimeInterval? maxAbsoluteError = targetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic); - private readonly OutlierMode outlierMode = targetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, resolver); - private readonly int minIterationCount = targetJob.ResolveValue(RunMode.MinIterationCountCharacteristic, resolver); - private readonly int maxIterationCount = targetJob.ResolveValue(RunMode.MaxIterationCountCharacteristic, resolver); - private int _iterationCounter = 0; + private readonly double maxRelativeError; + private readonly TimeInterval? maxAbsoluteError; + private readonly OutlierMode outlierMode; + private readonly int minIterationCount; + private readonly int maxIterationCount; + private readonly List measurementsForStatistics; + private int iterationCounter = 0; + + public EngineActualStageAuto(Job targetJob, IResolver resolver, IterationMode iterationMode) : base(iterationMode) + { + maxRelativeError = targetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, resolver); + maxAbsoluteError = targetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic); + outlierMode = targetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, resolver); + minIterationCount = targetJob.ResolveValue(RunMode.MinIterationCountCharacteristic, resolver); + maxIterationCount = targetJob.ResolveValue(RunMode.MaxIterationCountCharacteristic, resolver); + measurementsForStatistics = GetMeasurementList(); + } internal override List GetMeasurementList() => new (maxIterationCount); @@ -48,21 +59,23 @@ internal override bool GetShouldRunIteration(List measurements, ref const double MaxOverheadRelativeError = 0.05; bool isOverhead = Mode == IterationMode.Overhead; double effectiveMaxRelativeError = isOverhead ? MaxOverheadRelativeError : maxRelativeError; - _iterationCounter++; + iterationCounter++; + var measurement = measurements[measurements.Count - 1]; + measurementsForStatistics.Add(measurement); - var statistics = MeasurementsStatistics.Calculate(measurements, outlierMode); + var statistics = MeasurementsStatistics.Calculate(measurementsForStatistics, outlierMode); double actualError = statistics.LegacyConfidenceInterval.Margin; double maxError1 = effectiveMaxRelativeError * statistics.Mean; double maxError2 = maxAbsoluteError?.Nanoseconds ?? double.MaxValue; double maxError = Math.Min(maxError1, maxError2); - if (_iterationCounter >= minIterationCount && actualError < maxError) + if (iterationCounter >= minIterationCount && actualError < maxError) { return false; } - if (_iterationCounter >= maxIterationCount || isOverhead && _iterationCounter >= MaxOverheadIterationCount) + if (iterationCounter >= maxIterationCount || isOverhead && iterationCounter >= MaxOverheadIterationCount) { return false; }