Skip to content

Commit ff4fc43

Browse files
ZentrikZentrik
committed
Add some tests
1 parent 512ed0a commit ff4fc43

File tree

4 files changed

+140
-4
lines changed

4 files changed

+140
-4
lines changed

‎src/execution.jl

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -136,13 +136,20 @@ function _run(b::Benchmark, p::Parameters; verbose=false, pad="", warmup=true, k
136136
end
137137
if params.enable_customisable_func == :ALL
138138
params.customisable_gcsample && gcscrub()
139-
push!(trial.customisable_result, b.customisable_func(b.quote_vals, params)[1])
139+
s = b.customisable_func(b.quote_vals, params)
140+
push!(trial.customisable_result, s[1])
141+
142+
if params.run_customisable_func_only
143+
return_val = s[end]
144+
end
140145
end
141146

142147
iters = 2
143148
while (Base.time() - start_time) < params.seconds && iters ≤ params.samples
144-
params.gcsample && gcscrub()
145-
push!(trial, b.samplefunc(b.quote_vals, params)[1:(end - 1)]...)
149+
if !params.run_customisable_func_only
150+
params.gcsample && gcscrub()
151+
push!(trial, b.samplefunc(b.quote_vals, params)[1:(end - 1)]...)
152+
end
146153

147154
if params.enable_customisable_func == :ALL
148155
params.customisable_gcsample && gcscrub()
@@ -152,7 +159,7 @@ function _run(b::Benchmark, p::Parameters; verbose=false, pad="", warmup=true, k
152159
iters += 1
153160
end
154161

155-
if params.enable_customisable_func !== :FALSE
162+
if params.enable_customisable_func == :LAST
156163
params.customisable_gcsample && gcscrub()
157164
s = b.customisable_func(b.quote_vals, params)
158165
trial.customisable_result = s[1]

‎test/CustomisableBenchmarkTests.jl

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
module CustomisableBenchmarkTests
2+
3+
using BenchmarkTools
4+
using Test
5+
6+
x = Ref(0)
7+
setup_prehook(_) = x[] += 1
8+
prehook() = x[] += 1
9+
posthook() = x[] += 1
10+
function sample_result(_, setup_prehook_result, preehook_result, posthook_result)
11+
@test setup_prehook_result == 1
12+
@test preehook_result == 2
13+
@test posthook_result == 3
14+
@test x[] == 3
15+
return x[] += 1
16+
end
17+
function teardown_posthook(_, setup_prehook_result)
18+
@test setup_prehook_result == 1
19+
@test x[] == 4
20+
return x[] += 1
21+
end
22+
23+
@testset "Disabled custom benchmarking" begin
24+
x[] = 0
25+
res = @benchmark nothing setup_prehook = setup_prehook prehook = prehook posthook = posthook sample_result =
26+
sample_result teardown_posthook = teardown_posthook run_customisable_func_only = false
27+
@test res.customisable_result === nothing
28+
@test !res.customisable_result_for_every_sample
29+
end
30+
31+
@testset "custom benchmarking last" begin
32+
for run_customisable_func_only in (true, false)
33+
x[] = 0
34+
res = @benchmark nothing setup_prehook = setup_prehook prehook = prehook posthook =
35+
posthook sample_result = sample_result teardown_posthook = teardown_posthook enable_customisable_func =
36+
:LAST run_customisable_func_only = run_customisable_func_only
37+
if run_customisable_func_only
38+
@test isempty(res.times)
39+
@test isempty(res.gctimes)
40+
@test res.memory == typemax(Int)
41+
@test res.allocs == typemax(Int)
42+
end
43+
@test !res.customisable_result_for_every_sample
44+
@test res.customisable_result === 4
45+
end
46+
end
47+
48+
@testset "custom benchmark every sample, independent of iterations" begin
49+
for run_customisable_func_only in (true, false)
50+
x[] = 0
51+
setup_prehook(_) = x[] = 1
52+
res = @benchmark nothing setup_prehook = setup_prehook prehook = prehook posthook =
53+
posthook sample_result = sample_result teardown_posthook = teardown_posthook enable_customisable_func =
54+
:ALL run_customisable_func_only = run_customisable_func_only samples = 1000
55+
if run_customisable_func_only
56+
@test isempty(res.times)
57+
@test isempty(res.gctimes)
58+
@test res.memory == typemax(Int)
59+
@test res.allocs == typemax(Int)
60+
end
61+
@test res.customisable_result_for_every_sample
62+
@test res.customisable_result == fill(4, 1000)
63+
end
64+
end
65+
66+
@testset "custom benchmark every sample with iteration dependence" begin
67+
for run_customisable_func_only in (true, false)
68+
x[] = 0
69+
setup_prehook(_) = x[] += 1
70+
prehook() = x[] += 1
71+
posthook() = x[] += 1
72+
function sample_result(_, setup_prehook_result, preehook_result, posthook_result)
73+
return x[] += 1
74+
end
75+
function teardown_posthook(_, setup_prehook_result)
76+
return x[] += 1
77+
end
78+
res = @benchmark nothing setup_prehook = setup_prehook prehook = prehook posthook =
79+
posthook sample_result = sample_result teardown_posthook = teardown_posthook enable_customisable_func =
80+
:ALL run_customisable_func_only = run_customisable_func_only samples = 1000
81+
if run_customisable_func_only
82+
@test isempty(res.times)
83+
@test isempty(res.gctimes)
84+
@test res.memory == typemax(Int)
85+
@test res.allocs == typemax(Int)
86+
end
87+
@test res.customisable_result_for_every_sample
88+
@test res.customisable_result == collect(5 * (1:1000) .- 1)
89+
end
90+
end
91+
92+
end # module

‎test/ParametersTests.jl

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ BenchmarkTools.DEFAULT_PARAMETERS.gctrial = p.gctrial
1717
BenchmarkTools.DEFAULT_PARAMETERS.seconds = oldseconds
1818
BenchmarkTools.DEFAULT_PARAMETERS.gctrial = oldgctrial
1919

20+
f(x) = x
2021
p = Parameters(;
2122
seconds=1,
2223
gctrial=false,
@@ -26,6 +27,14 @@ p = Parameters(;
2627
gcsample=false,
2728
time_tolerance=0.043,
2829
memory_tolerance=0.15,
30+
# Customisable Parameters
31+
run_customisable_func_only=true,
32+
enable_customisable_func=:ALL,
33+
customisable_gcsample=true,
34+
# Customisable functions
35+
setup_prehook=f,
36+
teardown_posthook=f,
37+
sample_result=f,
2938
)
3039
oldseconds = BenchmarkTools.DEFAULT_PARAMETERS.seconds
3140
oldgctrial = BenchmarkTools.DEFAULT_PARAMETERS.gctrial
@@ -35,6 +44,16 @@ oldsamples = BenchmarkTools.DEFAULT_PARAMETERS.samples
3544
oldevals = BenchmarkTools.DEFAULT_PARAMETERS.evals
3645
oldoverhead = BenchmarkTools.DEFAULT_PARAMETERS.overhead
3746
oldgcsample = BenchmarkTools.DEFAULT_PARAMETERS.gcsample
47+
old_run_customisable_func_only =
48+
BenchmarkTools.DEFAULT_PARAMETERS.run_customisable_func_only
49+
old_enable_customisable_func = BenchmarkTools.DEFAULT_PARAMETERS.enable_customisable_func
50+
old_customisable_gcsample = BenchmarkTools.DEFAULT_PARAMETERS.customisable_gcsample
51+
old_setup_prehook = BenchmarkTools.DEFAULT_PARAMETERS.setup_prehook
52+
old_teardown_posthook = BenchmarkTools.DEFAULT_PARAMETERS.teardown_posthook
53+
old_sample_result = BenchmarkTools.DEFAULT_PARAMETERS.sample_result
54+
old_prehook = BenchmarkTools.DEFAULT_PARAMETERS.prehook
55+
old_posthook = BenchmarkTools.DEFAULT_PARAMETERS.posthook
56+
3857
BenchmarkTools.DEFAULT_PARAMETERS.seconds = p.seconds
3958
BenchmarkTools.DEFAULT_PARAMETERS.gctrial = p.gctrial
4059
BenchmarkTools.DEFAULT_PARAMETERS.time_tolerance = p.time_tolerance
@@ -43,6 +62,13 @@ BenchmarkTools.DEFAULT_PARAMETERS.samples = p.samples
4362
BenchmarkTools.DEFAULT_PARAMETERS.evals = p.evals
4463
BenchmarkTools.DEFAULT_PARAMETERS.overhead = p.overhead
4564
BenchmarkTools.DEFAULT_PARAMETERS.gcsample = p.gcsample
65+
BenchmarkTools.DEFAULT_PARAMETERS.run_customisable_func_only = p.run_customisable_func_only
66+
BenchmarkTools.DEFAULT_PARAMETERS.enable_customisable_func = p.enable_customisable_func
67+
BenchmarkTools.DEFAULT_PARAMETERS.customisable_gcsample = p.customisable_gcsample
68+
BenchmarkTools.DEFAULT_PARAMETERS.setup_prehook = p.setup_prehook
69+
BenchmarkTools.DEFAULT_PARAMETERS.teardown_posthook = p.teardown_posthook
70+
BenchmarkTools.DEFAULT_PARAMETERS.sample_result = p.sample_result
71+
4672
@test p == Parameters()
4773
@test p == Parameters(p)
4874
BenchmarkTools.DEFAULT_PARAMETERS.seconds = oldseconds
@@ -53,5 +79,12 @@ BenchmarkTools.DEFAULT_PARAMETERS.samples = oldsamples
5379
BenchmarkTools.DEFAULT_PARAMETERS.evals = oldevals
5480
BenchmarkTools.DEFAULT_PARAMETERS.overhead = oldoverhead
5581
BenchmarkTools.DEFAULT_PARAMETERS.gcsample = oldgcsample
82+
BenchmarkTools.DEFAULT_PARAMETERS.run_customisable_func_only =
83+
old_run_customisable_func_only
84+
BenchmarkTools.DEFAULT_PARAMETERS.enable_customisable_func = old_enable_customisable_func
85+
BenchmarkTools.DEFAULT_PARAMETERS.customisable_gcsample = old_customisable_gcsample
86+
BenchmarkTools.DEFAULT_PARAMETERS.setup_prehook = old_setup_prehook
87+
BenchmarkTools.DEFAULT_PARAMETERS.teardown_posthook = old_teardown_posthook
88+
BenchmarkTools.DEFAULT_PARAMETERS.sample_result = old_sample_result
5689

5790
end # module

‎test/runtests.jl

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,7 @@ println("done (took ", took_seconds, " seconds)")
3434
print("Testing serialization...")
3535
took_seconds = @elapsed include("SerializationTests.jl")
3636
println("done (took ", took_seconds, " seconds)")
37+
38+
print("Testing custom benchmarking...")
39+
took_seconds = @elapsed include("CustomisableBenchmarkTests.jl")
40+
println("done (took ", took_seconds, " seconds)")

0 commit comments

Comments
 (0)