Add postprocess::Function arg to benchmarkpkg() (#75)

* Add postprocess::Function arg to benchmarkpkg()

postprocess is a function that takes the results `Benchmarkgroup` and
either modifies them or returns a new version.

This allows users to inject logic into the benchmarking pipeline,
including subtracting two runs, dividing by N, or other such operations.

* Add unit test covering `postprocess=()->()`.

Author: NHDaly

src/runbenchmark.jl

@@ -10,6 +10,7 @@ The argument `pkg` can be a name of a package or a path to a directory to a pack
 **Keyword arguments**:
 
 * `script` - The script with the benchmarks, if not given, defaults to `benchmark/benchmarks.jl` in the package folder.
+* `postprocess` - A function to post-process results. Will be passed the `BenchmarkGroup`, which it can modify, or return a new one.
 * `resultfile` - If set, saves the output to `resultfile`
 * `retune` - Force a re-tune, saving the new tuning to the tune file.
 
@@ -28,12 +29,15 @@ benchmarkpkg(pathof(MyPkg), "my-feature"; script="/home/me/mycustombenchmark.jl"
 benchmarkpkg(pathof(MyPkg), BenchmarkConfig(id = "my-feature",
                                             env = Dict("JULIA_NUM_THREADS" => 4),
                                             juliacmd = `julia -O3`))
+benchmarkpkg(pathof(MyPkg),  # Run the benchmarks and divide the (median of) results by 1000
+    postprocess=(results)->(results["g"] = median(results["g"])/1_000)
 ```
 """
 function benchmarkpkg(
         pkg::String,
         target=BenchmarkConfig();
         script=nothing,
+        postprocess=nothing,
         resultfile=nothing,
         retune=false,
         custom_loadpath="" #= used in tests =#
@@ -90,6 +94,12 @@ function benchmarkpkg(
         io = IOBuffer(results_local["results"])
         seek(io, 0)
         resgroup = BenchmarkTools.load(io)[1]
+        if postprocess != nothing
+            retval = postprocess(resgroup)
+            if retval != nothing
+                resgroup = retval
+            end
+        end
         juliasha = results_local["juliasha"]
         vinfo = results_local["vinfo"]
         results = BenchmarkResults(pkg, shastring, resgroup, now(), juliasha, vinfo, target)

test/runtests.jl

@@ -1,5 +1,6 @@
 using PkgBenchmark
 using BenchmarkTools
+using Statistics
 using Test
 using Dates
 using LibGit2
@@ -73,6 +74,21 @@ temp_pkg_dir(;tmp_dir = tmp_dir) do
         end
     end
 
+    @testset "postprocess" begin
+        PkgBenchmark._withtemp(tempname()) do f
+            str = """
+            using BenchmarkTools
+            SUITE = BenchmarkGroup()
+            SUITE["foo"] = @benchmarkable for _ in 1:100; 1+1; end
+            """
+            open(f, "w") do file
+                print(file, str)
+            end
+            @test typeof(benchmarkpkg(TEST_PACKAGE_NAME, script=f;
+                postprocess=(r)->(r["foo"] = maximum(r["foo"]); return r))) == BenchmarkResults
+        end
+    end
+
     # Make a commit with a small benchmarks.jl file
     testpkg_path = Pkg.dir(TEST_PACKAGE_NAME)
     LibGit2.init(testpkg_path)