Skip to content

Commit bdb9abe

Browse files
AffieAffie
authored
Merge pull request #1708 from JuliaRobotics/23Q2/perf/benchmarks
Solver timing and start using BenchmarkTools
2 parents 845930c + 5b890d6 commit bdb9abe

File tree

4 files changed

+307
-0
lines changed

4 files changed

+307
-0
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,4 @@ Manifest.toml
88
/dev
99
*.cov
1010
lcov.info
11+
benchmark/results.md

‎benchmark/benchmarks.jl

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
2+
using BenchmarkTools
3+
4+
using IncrementalInference
5+
using RoME
6+
7+
##
8+
9+
# Define a parent BenchmarkGroup to contain our SUITE
10+
const SUITE = BenchmarkGroup()
11+
12+
# Add some child groups to our benchmark SUITE.
13+
SUITE["parametric"] = BenchmarkGroup(
14+
"1-init" => BenchmarkGroup([1, "par"]),
15+
"2-solve" => BenchmarkGroup([2, "par"]),
16+
"3-grow" => BenchmarkGroup([3, "par"]),
17+
)
18+
# SUITE["non-parametric"] = BenchmarkGroup(["2-solve"])
19+
# SUITE["construct"] = BenchmarkGroup()
20+
21+
SUITE["parametric"]["1-init"]["hex"] = @benchmarkable(
22+
IIF.autoinitParametric!(fg);
23+
samples = 2,
24+
seconds = 90,
25+
setup=(println("1-init fg"); fg=generateGraph_Hexagonal(;graphinit=false, landmark=false))
26+
)
27+
28+
SUITE["parametric"]["2-solve"]["hex"] = @benchmarkable(
29+
IIF.solveGraphParametric!(fg; init=false);
30+
samples = 2,
31+
seconds = 90,
32+
setup=(println("2-fg-1 solve"); fg=generateGraph_Hexagonal(;graphinit=false, landmark=false))
33+
)
34+
35+
SUITE["parametric"]["3-grow"]["hex"] = @benchmarkable(
36+
IIF.solveGraphParametric!(fg; init=false);
37+
samples = 2,
38+
seconds = 90,
39+
setup=(println("3-fg-2 solve"); fg=generateGraph_Hexagonal(;graphinit=false, landmark=true))
40+
)
41+
42+
SUITE["mmisam"] = BenchmarkGroup(
43+
"1-init" => BenchmarkGroup([1, "non-par"]),
44+
"2-solve" => BenchmarkGroup([2, "non-par"]),
45+
"3-grow" => BenchmarkGroup([3, "non-par"]),
46+
)
47+
48+
SUITE["mmisam"]["2-solve"]["hex"] = @benchmarkable(
49+
solveGraph!(fg);
50+
samples = 2,
51+
seconds = 90,
52+
setup=(println("fg-1 solve"); fg=generateGraph_Hexagonal(;graphinit=true, landmark=false))
53+
)
54+
55+
SUITE["mmisam"]["3-grow"]["hex"] = @benchmarkable(
56+
solveGraph!(fg);
57+
samples = 2,
58+
seconds = 90,
59+
setup=(println("fg-2 solve"); fg=generateGraph_Hexagonal(;graphinit=true, landmark=true))
60+
)
61+
62+
# TODO maintain order (numbered for now), it's a Dict so not guarantied
63+
leaves(SUITE)
64+
65+
# # If a cache of tuned parameters already exists, use it, otherwise, tune and cache
66+
# # the benchmark parameters. Reusing cached parameters is faster and more reliable
67+
# # than re-tuning `SUITE` every time the file is included.
68+
# paramspath = joinpath(dirname(@__FILE__), "params.json")
69+
70+
# if isfile(paramspath)
71+
# loadparams!(SUITE, BenchmarkTools.load(paramspath)[1], :evals);
72+
# else
73+
# tune!(SUITE)
74+
# BenchmarkTools.save(paramspath, BenchmarkTools.params(SUITE));
75+
# end

‎benchmark/runbenchmarks.jl

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
using PkgBenchmark
2+
3+
config =
4+
BenchmarkConfig(id=nothing, juliacmd=`julia -O3`, env=Dict("JULIA_NUM_THREADS" => 16))
5+
6+
results = benchmarkpkg("IncrementalInference", config; retune=false)
7+
export_markdown("benchmark/results.md", results)
8+
9+
10+
11+
12+
13+
14+
if false
15+
16+
result = run(SUITE["parametric"])
17+
18+
result = run(SUITE)
19+
20+
21+
foreach(leaves(result)) do bm
22+
printstyled("$(bm[1][1]) - $(bm[1][2])\n"; bold=true, reverse=true)
23+
display(bm[2])
24+
println()
25+
end
26+
27+
28+
end
29+
30+
31+
if false
32+
## Showing adding variables to fg re-compiles with parametric solve
33+
fg = initfg();
34+
fg.solverParams.graphinit=false;
35+
addVariable!(fg, :x0, Pose2);
36+
addFactor!(fg, [:x0], PriorPose2(MvNormal([0.0,0,0], diagm([0.1,0.1,0.01].^2))));
37+
38+
r = @timed IIF.solveGraphParametric!(fg; init=false);
39+
40+
timed = [r];
41+
42+
for i = 1:14
43+
fr = Symbol("x",i-1)
44+
to = Symbol("x",i)
45+
addVariable!(fg, to, Pose2)
46+
addFactor!(fg, [fr,to], Pose2Pose2(MvNormal([10.0,0,pi/3], diagm([0.5,0.5,0.05].^2))))
47+
r = @timed IIF.solveGraphParametric!(fg; init=false);
48+
push!(timed, r)
49+
end
50+
51+
52+
addVariable!(fg, :l1, RoME.Point2, tags=[:LANDMARK;]);
53+
addFactor!(fg, [:x0; :l1], Pose2Point2BearingRange(Normal(0.0,0.1), Normal(20.0, 1.0)));
54+
addFactor!(fg, [:x6; :l1], Pose2Point2BearingRange(Normal(0.0,0.1), Normal(20.0, 1.0)));
55+
56+
r = @timed IIF.solveGraphParametric!(fg; init=false);
57+
push!(timed, r);
58+
59+
getproperty.(timed, :time)
60+
61+
end
62+
63+
64+
65+

‎benchmark/tune.json

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
[
2+
{
3+
"Julia": "1.8.5",
4+
"BenchmarkTools": "1.0.0"
5+
},
6+
[
7+
[
8+
"BenchmarkGroup",
9+
{
10+
"data": {
11+
"parametric": [
12+
"BenchmarkGroup",
13+
{
14+
"data": {
15+
"1-init": [
16+
"BenchmarkGroup",
17+
{
18+
"data": {
19+
"hex": [
20+
"Parameters",
21+
{
22+
"gctrial": true,
23+
"time_tolerance": 0.05,
24+
"samples": 2,
25+
"evals": 1,
26+
"gcsample": false,
27+
"seconds": 90,
28+
"overhead": 0,
29+
"memory_tolerance": 0.01
30+
}
31+
]
32+
},
33+
"tags": [
34+
1,
35+
"par"
36+
]
37+
}
38+
],
39+
"2-solve": [
40+
"BenchmarkGroup",
41+
{
42+
"data": {
43+
"hex": [
44+
"Parameters",
45+
{
46+
"gctrial": true,
47+
"time_tolerance": 0.05,
48+
"samples": 2,
49+
"evals": 1,
50+
"gcsample": false,
51+
"seconds": 90,
52+
"overhead": 0,
53+
"memory_tolerance": 0.01
54+
}
55+
]
56+
},
57+
"tags": [
58+
2,
59+
"par"
60+
]
61+
}
62+
],
63+
"3-grow": [
64+
"BenchmarkGroup",
65+
{
66+
"data": {
67+
"hex": [
68+
"Parameters",
69+
{
70+
"gctrial": true,
71+
"time_tolerance": 0.05,
72+
"samples": 2,
73+
"evals": 1,
74+
"gcsample": false,
75+
"seconds": 90,
76+
"overhead": 0,
77+
"memory_tolerance": 0.01
78+
}
79+
]
80+
},
81+
"tags": [
82+
3,
83+
"par"
84+
]
85+
}
86+
]
87+
},
88+
"tags": [
89+
]
90+
}
91+
],
92+
"mmisam": [
93+
"BenchmarkGroup",
94+
{
95+
"data": {
96+
"1-init": [
97+
"BenchmarkGroup",
98+
{
99+
"data": {
100+
},
101+
"tags": [
102+
1,
103+
"non-par"
104+
]
105+
}
106+
],
107+
"2-solve": [
108+
"BenchmarkGroup",
109+
{
110+
"data": {
111+
"hex": [
112+
"Parameters",
113+
{
114+
"gctrial": true,
115+
"time_tolerance": 0.05,
116+
"samples": 2,
117+
"evals": 1,
118+
"gcsample": false,
119+
"seconds": 90,
120+
"overhead": 0,
121+
"memory_tolerance": 0.01
122+
}
123+
]
124+
},
125+
"tags": [
126+
2,
127+
"non-par"
128+
]
129+
}
130+
],
131+
"3-grow": [
132+
"BenchmarkGroup",
133+
{
134+
"data": {
135+
"hex": [
136+
"Parameters",
137+
{
138+
"gctrial": true,
139+
"time_tolerance": 0.05,
140+
"samples": 2,
141+
"evals": 1,
142+
"gcsample": false,
143+
"seconds": 90,
144+
"overhead": 0,
145+
"memory_tolerance": 0.01
146+
}
147+
]
148+
},
149+
"tags": [
150+
3,
151+
"non-par"
152+
]
153+
}
154+
]
155+
},
156+
"tags": [
157+
]
158+
}
159+
]
160+
},
161+
"tags": [
162+
]
163+
}
164+
]
165+
]
166+
]

0 commit comments

Comments
 (0)