Add more granular functions for time and memory judgements (#142)

* Add more granular functions for time and memory judgements

Author: DilumAluthge

src/groups.jl

@@ -82,15 +82,31 @@ judge(groups::BenchmarkGroup...; kwargs...) = mapvals((x...) -> judge(x...; kwar
 rmskew!(group::BenchmarkGroup) = mapvals!(rmskew!, group)
 rmskew(group::BenchmarkGroup) = mapvals(rmskew, group)
 
+isregression(f, group::BenchmarkGroup) = any((x) -> isregression(f, x), values(group))
 isregression(group::BenchmarkGroup) = any(isregression, values(group))
+
+isimprovement(f, group::BenchmarkGroup) = any((x) -> isimprovement(f, x), values(group))
 isimprovement(group::BenchmarkGroup) = any(isimprovement, values(group))
+
+isinvariant(f, group::BenchmarkGroup) = all((x) -> isinvariant(f, x), values(group))
 isinvariant(group::BenchmarkGroup) = all(isinvariant, values(group))
 
+invariants(f, x) = x
 invariants(x) = x
+
+regressions(f, x) = x
 regressions(x) = x
+
+improvements(f, x) = x
 improvements(x) = x
+
+invariants(f, group::BenchmarkGroup) = mapvals!((x) -> invariants(f, x), filtervals((x) -> isinvariant(f, x), group))
 invariants(group::BenchmarkGroup) = mapvals!(invariants, filtervals(isinvariant, group))
+
+regressions(f, group::BenchmarkGroup) = mapvals!((x) -> regressions(f, x), filtervals((x) -> isregression(f, x), group))
 regressions(group::BenchmarkGroup) = mapvals!(regressions, filtervals(isregression, group))
+
+improvements(f, group::BenchmarkGroup) = mapvals!((x) -> improvements(f, x), filtervals((x) -> isimprovement(f, x), group))
 improvements(group::BenchmarkGroup) = mapvals!(improvements, filtervals(isimprovement, group))
 
 function loadparams!(group::BenchmarkGroup, paramsgroup::BenchmarkGroup, fields...)

src/trials.jl

@@ -221,9 +221,14 @@ function judge(ratio::Real, tolerance::Float64)
     end
 end
 
-isimprovement(t::TrialJudgement) = time(t) == :improvement || memory(t) == :improvement
-isregression(t::TrialJudgement) = time(t) == :regression || memory(t) == :regression
-isinvariant(t::TrialJudgement) = time(t) == :invariant && memory(t) == :invariant
+isimprovement(f, t::TrialJudgement) = f(t) == :improvement
+isimprovement(t::TrialJudgement) = isimprovement(time, t) || isimprovement(memory, t)
+
+isregression(f, t::TrialJudgement) = f(t) == :regression
+isregression(t::TrialJudgement) = isregression(time, t) || isregression(memory, t)
+
+isinvariant(f, t::TrialJudgement) = f(t) == :invariant
+isinvariant(t::TrialJudgement) = isinvariant(time, t) && isinvariant(memory, t)
 
 const colormap = (
     regression = :red,

test/GroupsTests.jl

@@ -79,6 +79,26 @@ gtrial = BenchmarkGroup([], Dict("t" => trial))
        judge(g1, g2; time_tolerance = 0.1, memory_tolerance = 0.1))
 @test ratio(g1, g2) == ratio(judge(g1, g2))
 
+@test isinvariant(judge(g1, g1))
+@test isinvariant(time, judge(g1, g1))
+@test isinvariant(memory, judge(g1, g1))
+@test !(isregression(judge(g1, g1)))
+@test !(isregression(time, judge(g1, g1)))
+@test !(isregression(memory, judge(g1, g1)))
+@test !(isimprovement(judge(g1, g1)))
+@test !(isimprovement(time, judge(g1, g1)))
+@test !(isimprovement(memory, judge(g1, g1)))
+
+@test BenchmarkTools.invariants(judge(g1, g2)).data == Dict("c" => judge(tc, tc))
+@test BenchmarkTools.invariants(time, (judge(g1, g2))).data == Dict("c" => judge(tc, tc))
+@test BenchmarkTools.invariants(memory, (judge(g1, g2))).data == Dict("a" => judge(t1a, t2a), "b" => judge(t1b, t2b), "c" => judge(tc, tc))
+@test BenchmarkTools.regressions(judge(g1, g2)).data == Dict("b" => judge(t1b, t2b))
+@test BenchmarkTools.regressions(time, (judge(g1, g2))).data == Dict("b" => judge(t1b, t2b))
+@test BenchmarkTools.regressions(memory, (judge(g1, g2))).data == Dict()
+@test BenchmarkTools.improvements(judge(g1, g2)).data == Dict("a" => judge(t1a, t2a))
+@test BenchmarkTools.improvements(time, (judge(g1, g2))).data == Dict("a" => judge(t1a, t2a))
+@test BenchmarkTools.improvements(memory, (judge(g1, g2))).data == Dict()
+
 @test isinvariant(judge(g1, g1))
 @test !(isinvariant(judge(g1, g2)))
 @test isregression(judge(g1, g2))
@@ -89,6 +109,17 @@ gtrial = BenchmarkGroup([], Dict("t" => trial))
 @test regressions(judge(g1, g2)).data == Dict("b" => judge(t1b, t2b))
 @test improvements(judge(g1, g2)).data == Dict("a" => judge(t1a, t2a))
 
+struct Bar end
+@test BenchmarkTools.invariants(Bar()) == Bar()
+@test BenchmarkTools.invariants(time, (Bar())) == Bar()
+@test BenchmarkTools.invariants(memory, (Bar())) == Bar()
+@test BenchmarkTools.regressions(Bar()) == Bar()
+@test BenchmarkTools.regressions(time, (Bar())) == Bar()
+@test BenchmarkTools.regressions(memory, (Bar())) == Bar()
+@test BenchmarkTools.improvements(Bar()) == Bar()
+@test BenchmarkTools.improvements(time, (Bar())) == Bar()
+@test BenchmarkTools.improvements(memory, (Bar())) == Bar()
+
 @test minimum(gtrial)["t"] == minimum(gtrial["t"])
 @test median(gtrial)["t"] == median(gtrial["t"])
 @test mean(gtrial)["t"] == mean(gtrial["t"])

test/TrialsTests.jl

@@ -135,16 +135,46 @@ tj_r_2 = judge(tr; time_tolerance = 2.0, memory_tolerance = 2.0)
 @test isinvariant(tj_ab_2)
 @test isinvariant(tj_r_2)
 
+@test !(isinvariant(time, tj_ab))
+@test !(isinvariant(time, tj_r))
+@test isinvariant(time, tj_ab_2)
+@test isinvariant(time, tj_r_2)
+
+@test !(isinvariant(memory, tj_ab))
+@test !(isinvariant(memory, tj_r))
+@test isinvariant(memory, tj_ab_2)
+@test isinvariant(memory, tj_r_2)
+
 @test isregression(tj_ab)
 @test isregression(tj_r)
 @test !(isregression(tj_ab_2))
 @test !(isregression(tj_r_2))
 
+@test !(isregression(time, tj_ab))
+@test !(isregression(time, tj_r))
+@test !(isregression(time, tj_ab_2))
+@test !(isregression(time, tj_r_2))
+
+@test isregression(memory, tj_ab)
+@test isregression(memory, tj_r)
+@test !(isregression(memory, tj_ab_2))
+@test !(isregression(memory, tj_r_2))
+
 @test isimprovement(tj_ab)
 @test isimprovement(tj_r)
 @test !(isimprovement(tj_ab_2))
 @test !(isimprovement(tj_r_2))
 
+@test isimprovement(time, tj_ab)
+@test isimprovement(time, tj_r)
+@test !(isimprovement(time, tj_ab_2))
+@test !(isimprovement(time, tj_r_2))
+
+@test !(isimprovement(memory, tj_ab))
+@test !(isimprovement(memory, tj_r))
+@test !(isimprovement(memory, tj_ab_2))
+@test !(isimprovement(memory, tj_r_2))
+
 ###################
 # pretty printing #
 ###################