Skip to content

Commit 7d31ac8

Browse files
lkdvoslkdvos
committed
add plot script
1 parent 6e41eaa commit 7d31ac8

File tree

3 files changed

+303
-0
lines changed

3 files changed

+303
-0
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ Manifest.toml
33
.vscode
44
.DS_Store
55
**/dev/
6+
benchmark/results

‎benchmark/Project.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
33
BlockTensorKit = "5f87ffc2-9cf1-4a46-8172-465d160bd8cd"
44
CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"
5+
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
6+
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
57
MPSKit = "bb1c41ca-d63c-52ed-829e-0820dda26502"
68
MPSKitModels = "ca635005-6f8c-4cd1-b51d-8491250ef2ab"
79
TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"

‎benchmark/plot_results.jl‎

Lines changed: 300 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,300 @@
1+
using JSON
2+
using DataFrames
3+
using CairoMakie
4+
using Statistics
5+
6+
# Loading in the data
7+
# -------------------
8+
resultdir = joinpath(@__DIR__, "results")
9+
resultfile(i) = "results_MPSKit@bench$i.json"
10+
11+
df_contract = let df = DataFrame(
12+
:version => Int[], :model => String[], :symmetry => String[],
13+
:D => Int[], :V => Int[], :memory => Int[], :allocs => Int[], :times => Vector{Int}[]
14+
)
15+
16+
for version in 0:3
17+
result = JSON.parsefile(joinpath(resultdir, resultfile(version)))
18+
for (model, model_res) in result.data.derivatives.data.AC2_contraction.data
19+
for (symmetry, sym_res) in model_res.data
20+
for (DV, bench) in sym_res.data
21+
D, V = eval(Meta.parse(DV))::Tuple{Int, Int}
22+
23+
push!(
24+
df,
25+
(version, model, symmetry, D, V, bench.memory, bench.allocs, collect(Int, bench.times))
26+
)
27+
end
28+
end
29+
end
30+
end
31+
df
32+
end
33+
df_prep = let df = DataFrame(
34+
:version => Int[], :model => String[], :symmetry => String[],
35+
:D => Int[], :V => Int[], :memory => Int[], :allocs => Int[], :times => Vector{Int}[]
36+
)
37+
38+
for version in 0:3
39+
result = JSON.parsefile(joinpath(resultdir, resultfile(version)))
40+
for (model, model_res) in result.data.derivatives.data.AC2_preparation.data
41+
for (symmetry, sym_res) in model_res.data
42+
for (DV, bench) in sym_res.data
43+
D, V = eval(Meta.parse(DV))::Tuple{Int, Int}
44+
45+
push!(
46+
df,
47+
(version, model, symmetry, D, V, bench.memory, bench.allocs, collect(Int, bench.times))
48+
)
49+
end
50+
end
51+
end
52+
end
53+
df
54+
end
55+
56+
# Plotting the results
57+
# --------------------
58+
fontsize = 20
59+
estimator = median
60+
61+
f_times = let f = Figure(; size = (1400, 1400))
62+
models = ["heisenberg_nn", "heisenberg_nnn", "heisenberg_cylinder", "heisenberg_coulomb"]
63+
symmetries = ["Trivial", "Irrep[U₁]", "Irrep[SU₂]"]
64+
65+
66+
df_model = groupby(df_contract, [:model, :symmetry])
67+
for row in eachindex(models), col in eachindex(symmetries)
68+
df_data = get(df_model, (; model = models[row], symmetry = symmetries[col]), nothing)
69+
ax = Axis(f[row, col], xscale = log10, xlabel = "D", ylabel = "Δt (μs)", yscale = log10)
70+
@assert !isnothing(df_data)
71+
for (k, v) in pairs(groupby(df_data, :version))
72+
Ds = v[!, :D]
73+
times = estimator.(v[!, :times]) ./ 1.0e3
74+
I = sortperm(Ds)
75+
scatterlines!(ax, Ds[I], times[I]; label = "v$(k.version)")
76+
end
77+
axislegend(ax, position = :lt)
78+
end
79+
80+
Label(f[0, 0], "times"; fontsize)
81+
for (row, model) in enumerate(models)
82+
Label(f[row, 0], model; rotation = pi / 2, fontsize, tellheight = false, tellwidth = false)
83+
end
84+
for (col, symmetry) in enumerate(symmetries)
85+
Label(f[0, col], symmetry; fontsize, tellheight = false, tellwidth = false)
86+
end
87+
88+
f
89+
end
90+
save(joinpath(resultdir, "bench_times.png"), f_times)
91+
92+
f_times_relative = let f = Figure(; size = (1400, 1400))
93+
models = ["heisenberg_nn", "heisenberg_nnn", "heisenberg_cylinder", "heisenberg_coulomb"]
94+
symmetries = ["Trivial", "Irrep[U₁]", "Irrep[SU₂]"]
95+
96+
97+
df_model = groupby(df_contract, [:model, :symmetry])
98+
for row in eachindex(models), col in eachindex(symmetries)
99+
df_data = get(df_model, (; model = models[row], symmetry = symmetries[col]), nothing)
100+
ax = Axis(f[row, col], xscale = log10, xlabel = "D", ylabel = "Δt / Δt₀")
101+
hlines!([1], color = :red)
102+
@assert !isnothing(df_data)
103+
104+
df_v = groupby(df_data, :version)
105+
106+
v = get(df_v, (; version = 0), nothing)
107+
Ds = v[!, :D]
108+
times = estimator.(v[!, :times])
109+
I = sortperm(Ds)
110+
times₀ = times[I]
111+
112+
for (k, v) in pairs(groupby(df_data, :version))
113+
k.version == 0 && continue
114+
Ds = v[!, :D]
115+
I = sortperm(Ds)
116+
times = estimator.(v[!, :times])[I]
117+
scatterlines!(ax, Ds[I], times ./ times₀; label = "v$(k.version)")
118+
end
119+
axislegend(ax, position = :lt)
120+
end
121+
122+
Label(f[0, 0], "times"; fontsize)
123+
for (row, model) in enumerate(models)
124+
Label(f[row, 0], model; rotation = pi / 2, fontsize, tellheight = false, tellwidth = false)
125+
end
126+
for (col, symmetry) in enumerate(symmetries)
127+
Label(f[0, col], symmetry; fontsize, tellheight = false, tellwidth = false)
128+
end
129+
130+
f
131+
end
132+
save(joinpath(resultdir, "bench_times_relative.png"), f_times_relative)
133+
134+
f_allocs = let f = Figure(; size = (1400, 1400))
135+
models = ["heisenberg_nn", "heisenberg_nnn", "heisenberg_cylinder", "heisenberg_coulomb"]
136+
symmetries = ["Trivial", "Irrep[U₁]", "Irrep[SU₂]"]
137+
138+
139+
df_model = groupby(df_contract, [:model, :symmetry])
140+
for row in eachindex(models), col in eachindex(symmetries)
141+
df_data = get(df_model, (; model = models[row], symmetry = symmetries[col]), nothing)
142+
ax = Axis(f[row, col], xscale = log10, xlabel = "D", ylabel = "allocs", yscale = log10)
143+
@assert !isnothing(df_data)
144+
for (k, v) in pairs(groupby(df_data, :version))
145+
Ds = v[!, :D]
146+
allocs = estimator.(v[!, :allocs])
147+
I = sortperm(Ds)
148+
scatterlines!(ax, Ds[I], allocs[I]; label = "v$(k.version)")
149+
end
150+
axislegend(ax, position = :lt)
151+
end
152+
153+
Label(f[0, 0], "allocs"; fontsize)
154+
for (row, model) in enumerate(models)
155+
Label(f[row, 0], model; rotation = pi / 2, fontsize, tellheight = false, tellwidth = false)
156+
end
157+
for (col, symmetry) in enumerate(symmetries)
158+
Label(f[0, col], symmetry; fontsize, tellheight = false, tellwidth = false)
159+
end
160+
161+
f
162+
end
163+
save(joinpath(resultdir, "bench_allocs.png"), f_allocs)
164+
165+
f_memory = let f = Figure(; size = (1400, 1400))
166+
models = ["heisenberg_nn", "heisenberg_nnn", "heisenberg_cylinder", "heisenberg_coulomb"]
167+
symmetries = ["Trivial", "Irrep[U₁]", "Irrep[SU₂]"]
168+
169+
170+
df_model = groupby(df_contract, [:model, :symmetry])
171+
for row in eachindex(models), col in eachindex(symmetries)
172+
df_data = get(df_model, (; model = models[row], symmetry = symmetries[col]), nothing)
173+
ax = Axis(f[row, col], xscale = log10, xlabel = "D", ylabel = "memory (KiB)", yscale = log10)
174+
@assert !isnothing(df_data)
175+
for (k, v) in pairs(groupby(df_data, :version))
176+
Ds = v[!, :D]
177+
memory = estimator.(v[!, :memory]) ./ (2^10)
178+
I = sortperm(Ds)
179+
scatterlines!(ax, Ds[I], memory[I]; label = "v$(k.version)")
180+
end
181+
axislegend(ax, position = :lt)
182+
end
183+
184+
Label(f[0, 0], "memory"; fontsize)
185+
for (row, model) in enumerate(models)
186+
Label(f[row, 0], model; rotation = pi / 2, fontsize, tellheight = false, tellwidth = false)
187+
end
188+
for (col, symmetry) in enumerate(symmetries)
189+
Label(f[0, col], symmetry; fontsize, tellheight = false, tellwidth = false)
190+
end
191+
192+
f
193+
end
194+
save(joinpath(resultdir, "bench_memory.png"), f_allocs)
195+
196+
f_memory_relative = let f = Figure(; size = (1400, 1400))
197+
models = ["heisenberg_nn", "heisenberg_nnn", "heisenberg_cylinder", "heisenberg_coulomb"]
198+
symmetries = ["Trivial", "Irrep[U₁]", "Irrep[SU₂]"]
199+
200+
201+
df_model = groupby(df_contract, [:model, :symmetry])
202+
for row in eachindex(models), col in eachindex(symmetries)
203+
df_data = get(df_model, (; model = models[row], symmetry = symmetries[col]), nothing)
204+
ax = Axis(f[row, col], xscale = log10, xlabel = "D", ylabel = "memory / memory₀")
205+
hlines!([1], color = :red)
206+
@assert !isnothing(df_data)
207+
208+
df_v = groupby(df_data, :version)
209+
210+
v = get(df_v, (; version = 0), nothing)
211+
Ds = v[!, :D]
212+
times = estimator.(v[!, :memory])
213+
I = sortperm(Ds)
214+
times₀ = times[I]
215+
216+
for (k, v) in pairs(groupby(df_data, :version))
217+
k.version == 0 && continue
218+
Ds = v[!, :D]
219+
I = sortperm(Ds)
220+
times = estimator.(v[!, :memory])[I]
221+
scatterlines!(ax, Ds[I], times ./ times₀; label = "v$(k.version)")
222+
end
223+
axislegend(ax, position = :lt)
224+
end
225+
226+
Label(f[0, 0], "memory (relative)"; fontsize)
227+
for (row, model) in enumerate(models)
228+
Label(f[row, 0], model; rotation = pi / 2, fontsize, tellheight = false, tellwidth = false)
229+
end
230+
for (col, symmetry) in enumerate(symmetries)
231+
Label(f[0, col], symmetry; fontsize, tellheight = false, tellwidth = false)
232+
end
233+
234+
f
235+
end
236+
save(joinpath(resultdir, "bench_memory_relative.png"), f_memory_relative)
237+
238+
239+
# Including preparation times
240+
# ---------------------------
241+
for n_applications in [3, 10, 30]
242+
f_times_relative = let f = Figure(; size = (1400, 1400))
243+
models = ["heisenberg_nn", "heisenberg_nnn", "heisenberg_cylinder", "heisenberg_coulomb"]
244+
symmetries = ["Trivial", "Irrep[U₁]", "Irrep[SU₂]"]
245+
246+
247+
df_model = groupby(df_contract, [:model, :symmetry])
248+
dfp_model = groupby(df_prep, [:model, :symmetry])
249+
for row in eachindex(models), col in eachindex(symmetries)
250+
df_data = get(df_model, (; model = models[row], symmetry = symmetries[col]), nothing)
251+
dfp_data = get(dfp_model, (; model = models[row], symmetry = symmetries[col]), nothing)
252+
ax = Axis(f[row, col], xscale = log10, xlabel = "D", ylabel = "Δt / Δt₀")
253+
hlines!([1], color = :red)
254+
@assert !isnothing(df_data) && !isnothing(dfp_data)
255+
256+
df_v = groupby(df_data, :version)
257+
dfp_v = groupby(dfp_data, :version)
258+
259+
v = get(df_v, (; version = 0), nothing)
260+
Ds = v[!, :D]
261+
times = estimator.(v[!, :times])
262+
I = sortperm(Ds)
263+
times₀ = n_applications .* times[I]
264+
265+
vp = get(dfp_v, (; version = 0), nothing)
266+
Ds = vp[!, :D]
267+
times = estimator.(vp[!, :times])
268+
I = sortperm(Ds)
269+
times₀ .+= times[I]
270+
271+
df_data_v = groupby(dfp_data, :version)
272+
for (k, v) in pairs(groupby(df_data, :version))
273+
k.version == 0 && continue
274+
Ds = v[!, :D]
275+
I = sortperm(Ds)
276+
times = n_applications .* estimator.(v[!, :times])[I]
277+
278+
vp = get(df_data_v, (; k.version), nothing)
279+
@assert !isnothing(vp)
280+
Ds = vp[!, :D]
281+
I = sortperm(Ds)
282+
times .+= estimator.(vp[!, :times][I])
283+
284+
scatterlines!(ax, Ds[I], times ./ times₀; label = "v$(k.version)")
285+
end
286+
axislegend(ax, position = :lt)
287+
end
288+
289+
Label(f[0, 0], "times"; fontsize)
290+
for (row, model) in enumerate(models)
291+
Label(f[row, 0], model; rotation = pi / 2, fontsize, tellheight = false, tellwidth = false)
292+
end
293+
for (col, symmetry) in enumerate(symmetries)
294+
Label(f[0, col], symmetry; fontsize, tellheight = false, tellwidth = false)
295+
end
296+
297+
f
298+
end
299+
save(joinpath(resultdir, "bench_prep_times_relative_n=$n_applications.png"), f_times_relative)
300+
end

0 commit comments

Comments
 (0)