Add option to force warmup (or no warmup) (#177)

* Add warmup option

* Test save every result when warmup=false (and evals=1)

* Update docs, and precompilation, add tests

Author: LilithHafner

src/benchmarking.jl

@@ -13,12 +13,13 @@ function benchmark(init, setup, fs::Tuple{Vararg{Any, N}}, teardown;
         evals::Union{Int, Nothing}=nothing,
         samples::Union{Int, Nothing}=nothing,
         seconds::Union{Real, Nothing}=(samples===nothing ? DEFAULTS.seconds : 10*DEFAULTS.seconds)*N,
-        gc::Bool=DEFAULTS.gc) where N
-    _benchmark_1(init, setup, teardown, evals, samples, seconds, gc, fs...)
+        gc::Bool=DEFAULTS.gc,
+        warmup::Union{Bool, Nothing}=nothing) where N
+    _benchmark_1(init, setup, teardown, evals, samples, seconds, gc, warmup, fs...)
 end
-_benchmark_1(init, setup, teardown, evals::Union{Int, Nothing}, samples::Union{Int, Nothing}, seconds::Real, gc::Bool, fs...) =
-    _benchmark_1(init, setup, teardown, evals, samples, Float64(seconds), gc, fs...)
-function _benchmark_1(init, setup, teardown, evals::Union{Int, Nothing}, samples::Union{Int, Nothing}, seconds::Union{Float64, Nothing}, gc::Bool, fs...)
+_benchmark_1(init, setup, teardown, evals::Union{Int, Nothing}, samples::Union{Int, Nothing}, seconds::Real, gc::Bool, warmup::Union{Bool, Nothing}, fs...) =
+    _benchmark_1(init, setup, teardown, evals, samples, Float64(seconds), gc, warmup, fs...)
+function _benchmark_1(init, setup, teardown, evals::Union{Int, Nothing}, samples::Union{Int, Nothing}, seconds::Union{Float64, Nothing}, gc::Bool, warmup::Union{Bool, Nothing}, fs...)
     @nospecialize
     N = length(fs)
 
@@ -37,16 +38,24 @@ function _benchmark_1(init, setup, teardown, evals::Union{Int, Nothing}, samples
 
     samples == 0 && return ntuple(i -> Benchmark([_benchmark_2(args1, setup, teardown, gc, evals, false, fs...)[1][i]]), N)
 
-    warmup, start_time = _benchmark_2(args1, setup, teardown, gc, 1, false, fs...)
+    warmup_result, start_time = if warmup === false
+        if seconds == 0
+            _benchmark_2(args1, setup, teardown, gc, 1, true, fs...)
+        else
+            (nothing, time_ns())
+        end
+    else
+        _benchmark_2(args1, setup, teardown, gc, 1, false, fs...)
+    end
 
-    seconds == 0 && return ntuple(i -> Benchmark([warmup[i]]), N)
+    seconds == 0 && return ntuple(i -> Benchmark([warmup_result[i]]), N)
     new_evals = if evals === nothing
         @assert evals === samples === nothing && seconds !== nothing
 
-        if sum(w.time for w in warmup) > 2seconds && all(w.compile_fraction < .5 for w in warmup)
+        if warmup === nothing && sum(w.time for w in warmup_result) > 2seconds && all(w.compile_fraction < .5 for w in warmup_result)
             # The estimated runtime in the warmup already exceeds the time budget.
             # Return the warmup result (which is marked as not having a warmup).
-            return ntuple(i -> Benchmark([warmup[i]]), N)
+            return ntuple(i -> Benchmark([warmup_result[i]]), N)
         end
 
         calibration1, time = _benchmark_2(args1, setup, teardown, gc, 1, true, fs...)

src/precompile.jl

@@ -4,5 +4,5 @@ precompile(create_function, (Symbol,))
 precompile(create_first_function, (Symbol,))
 precompile(create_function, (Expr,))
 precompile(create_first_function, (Expr,))
-precompile(_benchmark_1, (Any,Any,Any,Int,Int,Float64,Bool,Any))
+precompile(_benchmark_1, (Any,Any,Any,Int,Int,Float64,Bool,Nothing,Any))
 precompile(Base.show, (IO, MIME"text/plain", Sample))

src/public.jl

@@ -113,6 +113,15 @@ arguments to ordinary functions. Keyword arguments to control executions are
     the benchmark. As an experimental option, this may be removed in the future or its
     semantics may change. This option also depends on Julia internals and so it may break in
     future versions of Julia.
+  - `warmup::Union{Nothing, Bool}` An experimental option to force warmups to run or not
+    run. With `warmup = true` we always run a warmup. With `warmup = false` we assume a
+    warmup has already been run and both skip a warmup and nevertheless report the first
+    sample as already having been warmed up. With `warmup=nothing` (the default) we almost
+    always run a warmup but will skip it if the estimated runtime of the warmup exceeds
+    twice the runtime budget and less than half of the warmup time is spent compiling. As
+    an experimental option, this may be removed in the future or its semantics may change.
+    The heuristic for choosing when to automatically skip the warmup may also change in
+    future versions of Chairmarks.
 
 # Interpolation
 
@@ -247,10 +256,10 @@ A global constant that holds default benchmarking parameters.
 When a parameter is unspecified it defaults to the value stored in `Chairmarks.DEFAULTS`.
 
 Currently there is one stable default: `Chairmarks.DEFAULTS.seconds::Float64` which defaults
-to 0.1; and one experimental default: `Chairmarks.DEFAULTS.gc::Bool` which defaults to
-`true`.
+to 0.1; and two experimental defaults: `Chairmarks.DEFAULTS.gc::Bool` which defaults to
+`true`, and `Chairmarks.DEFAULTS.warmup::Union{Bool, Nothing}` which defaults to `nothing`.
 
 All default values may be changed in the future and the `gc` default may be removed
 entirely.
 """
-const DEFAULTS = Defaults(0.1, true)
+const DEFAULTS = Defaults(0.1, true, nothing)

src/types.jl

@@ -84,4 +84,5 @@ The type of [`Chairmarks.DEFAULTS`](@ref)
 mutable struct Defaults
     seconds::Float64
     gc::Bool
+    warmup::Union{Bool, Nothing}
 end

test/runtests.jl

@@ -181,7 +181,7 @@ else
             @test GC.enable(true)
         end
 
-        @testset "no warmup" begin
+        @testset "no warmup heuristics" begin
             no_warmup_counter = Ref(0)
             res = @be begin no_warmup_counter[] += 1; sleep(.1) end seconds=.05
             @test no_warmup_counter[] == 1
@@ -192,6 +192,42 @@ else
             @test occursin("without a warmup", sprint(show, MIME"text/plain"(), res))
         end
 
+        @testset "no warmup parameter" begin
+            counter = Ref(0)
+            res = @be begin counter[] += 1; sleep(.1) end seconds=.05 warmup=true
+            @test counter[] == 2
+            sample = only(res.samples) # qualify only for compat
+            @test .1 < sample.time
+            @test sample.warmup == 1
+            @test !occursin("without a warmup", sprint(show, MIME"text/plain"(), sample))
+            @test !occursin("without a warmup", sprint(show, MIME"text/plain"(), res))
+
+            counter[] = 0
+            res = @be begin counter[] += 1; sleep(.1) end seconds=.05 warmup=false
+            @test counter[] == 1
+            sample = only(res.samples) # qualify only for compat
+            @test .1 < sample.time
+            @test sample.warmup == 1
+            @test !occursin("without a warmup", sprint(show, MIME"text/plain"(), sample))
+            @test !occursin("without a warmup", sprint(show, MIME"text/plain"(), res))
+
+            counter[] = 0
+            res = @be begin counter[] += 1; sleep(.001) end seconds=.05 warmup=false
+            # @test counter[] > 1 This would be flaky
+            @test .001 < minimum(res).time
+            @test 1 == only(unique(s.evals for s in res.samples))
+            @test all(s -> s.warmup == 1, res.samples)
+            @test length(res.samples) == counter[] # Save and return every sample
+
+            counter[] = 0
+            res = @be begin counter[] += 1; sleep(.001) end seconds=0 warmup=false # warmup=false and seconds=0 is a notable edge case.
+            @test counter[] == 1
+            sample = only(res.samples) # qualify only for compat
+            @test sample.evals == 1
+            @test .001 < sample.time
+            @test sample.warmup == 1
+        end
+
         @testset "writefixed" begin
             @test Chairmarks.writefixed(-1.23045, 4) == "-1.2305"
             @test Chairmarks.writefixed(-1.23045, 3) == "-1.230"