Engine.Run() should return the full list of performed measurements (fixes #2187) (#2188)

* Engine.Run() should return the full list of performed measurements (fixes #2187)

* Code review fixes

Author: AndreyAkinshin

src/BenchmarkDotNet/Engines/Engine.cs

@@ -41,6 +41,7 @@ public class Engine : IEngine
         private bool EvaluateOverhead { get; }
         private bool MemoryRandomization { get; }
 
+        private readonly List<Measurement> jittingMeasurements = new (10);
         private readonly EnginePilotStage pilotStage;
         private readonly EngineWarmupStage warmupStage;
         private readonly EngineActualStage actualStage;
@@ -104,8 +105,10 @@ public void Dispose()
 
         public RunResults Run()
         {
+            var measurements = new List<Measurement>();
+            measurements.AddRange(jittingMeasurements);
+
             long invokeCount = TargetJob.ResolveValue(RunMode.InvocationCountCharacteristic, Resolver, 1);
-            IReadOnlyList<Measurement> idle = null;
 
             if (EngineEventSource.Log.IsEnabled())
                 EngineEventSource.Log.BenchmarkStart(BenchmarkName);
@@ -114,21 +117,23 @@ public RunResults Run()
             {
                 if (Strategy != RunStrategy.Monitoring)
                 {
-                    invokeCount = pilotStage.Run();
+                    var pilotStageResult = pilotStage.Run();
+                    invokeCount = pilotStageResult.PerfectInvocationCount;
+                    measurements.AddRange(pilotStageResult.Measurements);
 
                     if (EvaluateOverhead)
                     {
-                        warmupStage.RunOverhead(invokeCount, UnrollFactor);
-                        idle = actualStage.RunOverhead(invokeCount, UnrollFactor);
+                        measurements.AddRange(warmupStage.RunOverhead(invokeCount, UnrollFactor));
+                        measurements.AddRange(actualStage.RunOverhead(invokeCount, UnrollFactor));
                     }
                 }
 
-                warmupStage.RunWorkload(invokeCount, UnrollFactor, Strategy);
+                measurements.AddRange(warmupStage.RunWorkload(invokeCount, UnrollFactor, Strategy));
             }
 
             Host.BeforeMainRun();
 
-            var main = actualStage.RunWorkload(invokeCount, UnrollFactor, forceSpecific: Strategy == RunStrategy.Monitoring);
+            measurements.AddRange(actualStage.RunWorkload(invokeCount, UnrollFactor, forceSpecific: Strategy == RunStrategy.Monitoring));
 
             Host.AfterMainRun();
 
@@ -141,7 +146,7 @@ public RunResults Run()
 
             var outlierMode = TargetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, Resolver);
 
-            return new RunResults(idle, main, outlierMode, workGcHasDone, threadingStats, exceptionFrequency);
+            return new RunResults(measurements, outlierMode, workGcHasDone, threadingStats, exceptionFrequency);
         }
 
         public Measurement RunIteration(IterationData data)
@@ -183,6 +188,8 @@ public Measurement RunIteration(IterationData data)
             // Results
             var measurement = new Measurement(0, data.IterationMode, data.IterationStage, data.Index, totalOperations, clockSpan.GetNanoseconds());
             WriteLine(measurement.ToString());
+            if (measurement.IterationStage == IterationStage.Jitting)
+                jittingMeasurements.Add(measurement);
 
             Consume(stackMemory);
 

src/BenchmarkDotNet/Engines/EnginePilotStage.cs

@@ -1,13 +1,35 @@
 using System;
+using System.Collections.Generic;
 using BenchmarkDotNet.Environments;
 using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Reports;
+using JetBrains.Annotations;
 using Perfolizer.Horology;
 
 namespace BenchmarkDotNet.Engines
 {
     // TODO: use clockResolution
     internal class EnginePilotStage : EngineStage
     {
+        public readonly struct PilotStageResult
+        {
+            public long PerfectInvocationCount { get; }
+            [NotNull]
+            public IReadOnlyList<Measurement> Measurements { get; }
+
+            public PilotStageResult(long perfectInvocationCount, [NotNull] List<Measurement> measurements)
+            {
+                PerfectInvocationCount = perfectInvocationCount;
+                Measurements = measurements;
+            }
+
+            public PilotStageResult(long perfectInvocationCount)
+            {
+                PerfectInvocationCount = perfectInvocationCount;
+                Measurements = Array.Empty<Measurement>();
+            }
+        }
+
         internal const long MaxInvokeCount = (long.MaxValue / 2 + 1) / 2;
 
         private readonly int unrollFactor;
@@ -30,11 +52,11 @@ public EnginePilotStage(IEngine engine) : base(engine)
         }
 
         /// <returns>Perfect invocation count</returns>
-        public long Run()
+        public PilotStageResult Run()
         {
             // If InvocationCount is specified, pilot stage should be skipped
             if (TargetJob.HasValue(RunMode.InvocationCountCharacteristic))
-                return TargetJob.Run.InvocationCount;
+                return new PilotStageResult(TargetJob.Run.InvocationCount);
 
             // Here we want to guess "perfect" amount of invocation
             return TargetJob.HasValue(RunMode.IterationTimeCharacteristic)
@@ -45,15 +67,17 @@ public long Run()
         /// <summary>
         /// A case where we don't have specific iteration time.
         /// </summary>
-        private long RunAuto()
+        private PilotStageResult RunAuto()
         {
             long invokeCount = Autocorrect(minInvokeCount);
+            var measurements = new List<Measurement>();
 
             int iterationCounter = 0;
             while (true)
             {
                 iterationCounter++;
                 var measurement = RunIteration(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, unrollFactor);
+                measurements.Add(measurement);
                 double iterationTime = measurement.Nanoseconds;
                 double operationError = 2.0 * resolution / invokeCount; // An operation error which has arisen due to the Chronometer precision
 
@@ -75,15 +99,16 @@ private long RunAuto()
             }
             WriteLine();
 
-            return invokeCount;
+            return new PilotStageResult(invokeCount, measurements);
         }
 
         /// <summary>
         /// A case where we have specific iteration time.
         /// </summary>
-        private long RunSpecific()
+        private PilotStageResult RunSpecific()
         {
             long invokeCount = Autocorrect(Engine.MinInvokeCount);
+            var measurements = new List<Measurement>();
 
             int iterationCounter = 0;
 
@@ -92,6 +117,7 @@ private long RunSpecific()
             {
                 iterationCounter++;
                 var measurement = RunIteration(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, unrollFactor);
+                measurements.Add(measurement);
                 double actualIterationTime = measurement.Nanoseconds;
                 long newInvokeCount = Autocorrect(Math.Max(minInvokeCount, (long)Math.Round(invokeCount * targetIterationTime / actualIterationTime)));
 
@@ -105,7 +131,7 @@ private long RunSpecific()
             }
             WriteLine();
 
-            return invokeCount;
+            return new PilotStageResult(invokeCount, measurements);
         }
 
         private long Autocorrect(long count) => (count + unrollFactor - 1) / unrollFactor * unrollFactor;

src/BenchmarkDotNet/Engines/EngineWarmupStage.cs

@@ -9,13 +9,13 @@ internal class EngineWarmupStage : EngineStage
 
         public EngineWarmupStage(IEngine engine) : base(engine) => this.engine = engine;
 
-        public void RunOverhead(long invokeCount, int unrollFactor)
+        public IReadOnlyList<Measurement> RunOverhead(long invokeCount, int unrollFactor)
             => Run(invokeCount, IterationMode.Overhead, unrollFactor, RunStrategy.Throughput);
 
-        public void RunWorkload(long invokeCount, int unrollFactor, RunStrategy runStrategy)
+        public IReadOnlyList<Measurement> RunWorkload(long invokeCount, int unrollFactor, RunStrategy runStrategy)
             => Run(invokeCount, IterationMode.Workload, unrollFactor, runStrategy);
 
-        internal List<Measurement> Run(long invokeCount, IterationMode iterationMode, int unrollFactor, RunStrategy runStrategy)
+        internal IReadOnlyList<Measurement> Run(long invokeCount, IterationMode iterationMode, int unrollFactor, RunStrategy runStrategy)
         {
             var criteria = DefaultStoppingCriteriaFactory.Instance.CreateWarmup(engine.TargetJob, engine.Resolver, iterationMode, runStrategy);
             return Run(criteria, invokeCount, iterationMode, IterationStage.Warmup, unrollFactor);

src/BenchmarkDotNet/Engines/RunResults.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using BenchmarkDotNet.Extensions;
 using BenchmarkDotNet.Mathematics;
 using BenchmarkDotNet.Reports;
 using JetBrains.Annotations;
@@ -13,39 +14,51 @@ public struct RunResults
     {
         private readonly OutlierMode outlierMode;
 
+        [NotNull, PublicAPI]
+        public IReadOnlyList<Measurement> EngineMeasurements { get; }
+
         [CanBeNull, PublicAPI]
-        public IReadOnlyList<Measurement> Overhead { get; }
+        public IReadOnlyList<Measurement> Overhead
+            => EngineMeasurements
+                .Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual))
+                .ToArray();
 
         [NotNull, PublicAPI]
-        public IReadOnlyList<Measurement> Workload { get; }
+        public IReadOnlyList<Measurement> Workload
+            => EngineMeasurements
+                .Where(m => m.Is(IterationMode.Workload, IterationStage.Actual))
+                .ToArray();
 
         public GcStats GCStats { get; }
 
         public ThreadingStats ThreadingStats { get; }
 
         public double ExceptionFrequency { get; }
 
-        public RunResults([CanBeNull] IReadOnlyList<Measurement> overhead,
-                          [NotNull] IReadOnlyList<Measurement> workload,
-                          OutlierMode outlierMode,
-                          GcStats gcStats,
-                          ThreadingStats threadingStats,
-                          double exceptionFrequency)
+        public RunResults([NotNull] IReadOnlyList<Measurement> engineMeasurements,
+            OutlierMode outlierMode,
+            GcStats gcStats,
+            ThreadingStats threadingStats,
+            double exceptionFrequency)
         {
             this.outlierMode = outlierMode;
-            Overhead = overhead;
-            Workload = workload;
+            EngineMeasurements = engineMeasurements;
             GCStats = gcStats;
             ThreadingStats = threadingStats;
             ExceptionFrequency = exceptionFrequency;
         }
 
-        public IEnumerable<Measurement> GetMeasurements()
+        public IEnumerable<Measurement> GetWorkloadResultMeasurements()
         {
-            double overhead = Overhead == null ? 0.0 : new Statistics(Overhead.Select(m => m.Nanoseconds)).Median;
-            var mainStats = new Statistics(Workload.Select(m => m.Nanoseconds));
+            var overheadActualMeasurements = Overhead ?? Array.Empty<Measurement>();
+            var workloadActualMeasurements = Workload;
+            if (workloadActualMeasurements.IsEmpty())
+                yield break;
+
+            double overhead = overheadActualMeasurements.IsEmpty() ? 0.0 : new Statistics(overheadActualMeasurements.Select(m => m.Nanoseconds)).Median;
+            var mainStats = new Statistics(workloadActualMeasurements.Select(m => m.Nanoseconds));
             int resultIndex = 0;
-            foreach (var measurement in Workload)
+            foreach (var measurement in workloadActualMeasurements)
             {
                 if (mainStats.IsActualOutlier(measurement.Nanoseconds, outlierMode))
                     continue;
@@ -63,9 +76,17 @@ public IEnumerable<Measurement> GetMeasurements()
             }
         }
 
+        public IEnumerable<Measurement> GetAllMeasurements()
+        {
+            foreach (var measurement in EngineMeasurements)
+                yield return measurement;
+            foreach (var measurement in GetWorkloadResultMeasurements())
+                yield return measurement;
+        }
+
         public void Print(TextWriter outWriter)
         {
-            foreach (var measurement in GetMeasurements())
+            foreach (var measurement in GetWorkloadResultMeasurements())
                 outWriter.WriteLine(measurement.ToString());
 
             if (!GCStats.Equals(GcStats.Empty))

src/BenchmarkDotNet/Toolchains/Results/ExecuteResult.cs

@@ -67,7 +67,7 @@ internal ExecuteResult(List<Measurement> measurements, GcStats gcStats, Threadin
         internal static ExecuteResult FromRunResults(RunResults runResults, int exitCode)
             => exitCode != 0
                 ? CreateFailed(exitCode)
-                : new ExecuteResult(runResults.GetMeasurements().ToList(), runResults.GCStats, runResults.ThreadingStats, runResults.ExceptionFrequency);
+                : new ExecuteResult(runResults.GetAllMeasurements().ToList(), runResults.GCStats, runResults.ThreadingStats, runResults.ExceptionFrequency);
 
         internal static ExecuteResult CreateFailed(int exitCode = -1)
             => new ExecuteResult(false, exitCode, default, Array.Empty<string>(), Array.Empty<string>(), Array.Empty<string>(), 0);

tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs

@@ -72,8 +72,11 @@ public RunResults Run()
                 Console.WriteLine(EngineRunMessage);
 
                 return new RunResults(
-                    new List<Measurement> { new Measurement(1, IterationMode.Overhead, IterationStage.Actual, 1, 1, 1) },
-                    new List<Measurement> { new Measurement(1, IterationMode.Workload, IterationStage.Actual, 1, 1, 1) },
+                    new List<Measurement>
+                    {
+                        new (1, IterationMode.Overhead, IterationStage.Actual, 1, 1, 1),
+                        new (1, IterationMode.Workload, IterationStage.Actual, 1, 1, 1)
+                    },
                     OutlierMode.DontRemove,
                     default,
                     default,

tests/BenchmarkDotNet.IntegrationTests/EngineTests.cs

@@ -0,0 +1,78 @@
+using System.Linq;
+using System.Threading;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Engines;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Reports;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace BenchmarkDotNet.IntegrationTests
+{
+    public class EngineTests : BenchmarkTestExecutor
+    {
+        public EngineTests(ITestOutputHelper output) : base(output) { }
+
+        [Fact]
+        public void ZeroWarmupCountIsApplied()
+        {
+            var job = Job.InProcess
+                .WithEvaluateOverhead(false)
+                .WithWarmupCount(0)
+                .WithIterationCount(1)
+                .WithInvocationCount(1)
+                .WithUnrollFactor(1);
+            var config = DefaultConfig.Instance.AddJob(job).WithOptions(ConfigOptions.DisableOptimizationsValidator);
+            var summary = CanExecute<FooBench>(config);
+            var report = summary.Reports.Single();
+            int workloadWarmupCount = report.AllMeasurements
+                .Count(m => m.Is(IterationMode.Workload, IterationStage.Warmup));
+            Assert.Equal(0, workloadWarmupCount);
+        }
+
+        [Fact]
+        public void AllMeasurementsArePerformedDefault() => AllMeasurementsArePerformed(Job.Default);
+
+        [Fact]
+        public void AllMeasurementsArePerformedInProcess() => AllMeasurementsArePerformed(Job.InProcess);
+
+        private void AllMeasurementsArePerformed(Job baseJob)
+        {
+            var job = baseJob
+                .WithWarmupCount(1)
+                .WithIterationCount(1)
+                .WithInvocationCount(1)
+                .WithUnrollFactor(1);
+            var config = DefaultConfig.Instance.AddJob(job).WithOptions(ConfigOptions.DisableOptimizationsValidator);
+            var summary = CanExecute<FooBench>(config);
+            var measurements = summary.Reports.Single().AllMeasurements;
+
+            Output.WriteLine("*** AllMeasurements ***");
+            foreach (var measurement in measurements)
+                Output.WriteLine(measurement.ToString());
+            Output.WriteLine("-----");
+
+            void Check(IterationMode mode, IterationStage stage)
+            {
+                int count = measurements.Count(m => m.Is(mode, stage));
+                Output.WriteLine($"Count({mode}{stage}) = {count}");
+                Assert.True(count > 0, $"AllMeasurements don't contain {mode}{stage}");
+            }
+
+            Check(IterationMode.Overhead, IterationStage.Jitting);
+            Check(IterationMode.Workload, IterationStage.Jitting);
+            Check(IterationMode.Overhead, IterationStage.Warmup);
+            Check(IterationMode.Overhead, IterationStage.Actual);
+            Check(IterationMode.Workload, IterationStage.Warmup);
+            Check(IterationMode.Workload, IterationStage.Actual);
+            Check(IterationMode.Workload, IterationStage.Result);
+        }
+
+        public class FooBench
+        {
+            [Benchmark]
+            public void Foo() => Thread.Sleep(10);
+        }
+    }
+}