Merge pull request #255 from JuliaCI/teh/histrange

Support manual range for hist display

Author: vchuravy

docs/src/index.md

@@ -37,6 +37,9 @@ julia> @btime sin(x) setup=(x=rand())
 0.49587200950472454
 ```
 
+If you're interested in profiling a fast-running command, you can use `@bprofile sin(x) setup=(x=rand())` and then your favorite
+tools for displaying the results (`Profile.print` or a graphical viewer).
+
 If the expression you want to benchmark depends on external variables, you should use [`$` to "interpolate"](https://github.com/JuliaCI/BenchmarkTools.jl/blob/master/doc/manual.md#interpolating-values-into-benchmark-expressions) them into the benchmark expression to
 [avoid the problems of benchmarking with globals](https://docs.julialang.org/en/v1/manual/performance-tips/#Avoid-global-variables).
 Essentially, any interpolated variable `$x` or expression `$(...)` is "pre-computed" before benchmarking begins:

docs/src/manual.md

@@ -896,6 +896,42 @@ julia> loadparams!(suite, BenchmarkTools.load("params.json")[1], :evals, :sample
 Caching parameters in this manner leads to a far shorter turnaround time, and more importantly, much more consistent results.
 
 ## Visualizing benchmark results
+
+For comparing two or more benchmarks against one another, you can manually specify the range of the histogram using an
+`IOContext` to set `:histmin` and `:histmax`:
+
+```julia
+julia> io = IOContext(stdout, :histmin=>0.5, :histmax=>8, :logbins=>true)
+IOContext(Base.TTY(RawFD(13) open, 0 bytes waiting))
+
+julia> b = @benchmark x^3   setup=(x = rand()); show(io, MIME("text/plain"), b)
+BenchmarkTools.Trial: 10000 samples with 1000 evaluations.
+ Range (min … max):  1.239 ns … 31.433 ns  ┊ GC (min … max): 0.00% … 0.00%
+ Time  (median):     1.244 ns              ┊ GC (median):    0.00%
+ Time  (mean ± σ):   1.266 ns ±  0.611 ns  ┊ GC (mean ± σ):  0.00% ± 0.00%
+
+       █
+  ▁▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ ▂
+  0.5 ns       Histogram: log(frequency) by time        8 ns <
+
+ Memory estimate: 0 bytes, allocs estimate: 0.
+julia> b = @benchmark x^3.0 setup=(x = rand()); show(io, MIME("text/plain"), b)
+BenchmarkTools.Trial: 10000 samples with 1000 evaluations.
+ Range (min … max):  5.636 ns … 38.756 ns  ┊ GC (min … max): 0.00% … 0.00%
+ Time  (median):     5.662 ns              ┊ GC (median):    0.00%
+ Time  (mean ± σ):   5.767 ns ±  1.384 ns  ┊ GC (mean ± σ):  0.00% ± 0.00%
+
+                                         █▆    ▂             ▁
+  ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁███▄▄▃█▁▁▁▁▁▁▁▁▁▁▁▁ █
+  0.5 ns       Histogram: log(frequency) by time        8 ns <
+
+ Memory estimate: 0 bytes, allocs estimate: 0.
+
+```
+
+Set `:logbins` to `true` or `false` to ensure that all use the same vertical scaling (log frequency or frequency).
+
+
 The `Trial` object can be visualized using the `BenchmarkPlots` package:
 
 ```julia

src/trials.jl

@@ -290,8 +290,7 @@ function withtypename(f, io, t)
     end
 end
 
-function bindata(sorteddata, nbins)
-    min, max = sorteddata[[1; end]]
+function bindata(sorteddata, nbins, min, max)
     Δ = (max - min) / nbins
     bins = zeros(nbins)
     lastpos = 0
@@ -302,6 +301,7 @@ function bindata(sorteddata, nbins)
     end
     bins
 end
+bindata(sorteddata, nbins) = bindata(sorteddata, nbins, first(sorteddata), last(sorteddata))
 
 function asciihist(bins, height=1)
     histbars = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█']
@@ -440,20 +440,25 @@ function Base.show(io::IO, ::MIME"text/plain", t::Trial)
     histwidth = 42 + lmaxtimewidth + rmaxtimewidth
 
     histtimes = times[1:round(Int, histquantile*end)]
-    bins, logbins = bindata(histtimes, histwidth - 1), false
+    histmin = get(io, :histmin, first(histtimes))
+    histmax = get(io, :histmax, last(histtimes))
+    logbins = get(io, :logbins, nothing)
+    bins = bindata(histtimes, histwidth - 1, histmin, histmax)
     append!(bins, [1, floor((1-histquantile) * length(times))])
     # if median size of (bins with >10% average data/bin) is less than 5% of max bin size, log the bin sizes
-    if median(filter(b -> b > 0.1 * length(times) / histwidth, bins)) / maximum(bins) < 0.05
+    if (logbins === nothing || logbins === true) && median(filter(b -> b > 0.1 * length(times) / histwidth, bins)) / maximum(bins) < 0.05
         bins, logbins = log.(1 .+ bins), true
+    elseif logbins === nothing
+        logbins = false
     end
     hist = asciihist(bins, histheight)
     hist[:,end-1] .= ' '
     maxbin = maximum(bins)
 
-    delta1 = (histtimes[end] - histtimes[1]) / (histwidth - 1)
+    delta1 = (histmax - histmin) / (histwidth - 1)
     if delta1 > 0
-        medpos = 1 + round(Int, (histtimes[length(times) ÷ 2] - histtimes[1]) / delta1)
-        avgpos = 1 + round(Int, (mean(times) - histtimes[1]) / delta1)
+        medpos = 1 + round(Int, (histtimes[length(times) ÷ 2] - histmin) / delta1)
+        avgpos = 1 + round(Int, (mean(times) - histmin) / delta1)
     else
         medpos, avgpos = 1, 1
     end
@@ -470,7 +475,7 @@ function Base.show(io::IO, ::MIME"text/plain", t::Trial)
     end
 
     remtrailingzeros(timestr) = replace(timestr, r"\.?0+ " => " ")
-    minhisttime, maxhisttime = remtrailingzeros.(prettytime.(round.(histtimes[[1; end]], sigdigits=3)))
+    minhisttime, maxhisttime = remtrailingzeros.(prettytime.(round.([histmin, histmax], sigdigits=3)))
 
     print(io, "\n", pad, "  ", minhisttime)
     caption = "Histogram: " * ( logbins ? "log(frequency)" : "frequency" ) * " by time"

test/ExecutionTests.jl

@@ -135,6 +135,16 @@ tune!(b)
 # test kwargs separated by `,`
 @benchmark(output=sin(x), setup=(x=1.0; output=0.0), teardown=(@test output == sin(x)))
 
+io = IOBuffer()
+ioctx = IOContext(io, :histmin=>0.5, :histmax=>8, :logbins=>false)
+b = @benchmark x^3   setup=(x = rand()); show(ioctx, MIME("text/plain"), b)
+b = @benchmark x^3.0 setup=(x = rand()); show(ioctx, MIME("text/plain"), b)
+str = String(take!(io))
+idx = findfirst(r"0.5 ns +Histogram: frequency by time +8 ns", str)
+@test isa(idx, UnitRange)
+idx = findnext( r"0.5 ns +Histogram: frequency by time +8 ns", str, idx[end]+1)
+@test isa(idx, UnitRange)
+
 #############
 # @bprofile #
 #############