Test workaround for BenchmarkTools memory leak, delete src/zerl.jl.

Author: chriselrod

benchmark/benchmarkflops.jl

@@ -3,7 +3,7 @@ include(joinpath(LOOPVECBENCHDIR, "loadsharedlibs.jl"))
 
 using BenchmarkTools, SharedArrays
 struct SizedResults{V <: AbstractVector} <: AbstractMatrix{String}
-    results::SharedMatrix{Float64}
+    results::Matrix{Float64}
     sizes::V
 end
 function Base.size(sr::SizedResults)
@@ -14,11 +14,11 @@ struct BenchmarkResult{V}
     tests::Vector{String}
     sizedresults::SizedResults{V}
 end
-function BenchmarkResult(tests, sizes)
+function BenchmarkResult(results, tests, sizes)
     ntests = length(tests); nsizes = length(sizes)
     BenchmarkResult(
         append!(["Size"], tests),
-        SizedResults(SharedMatrix{Float64}(ntests, nsizes), sizes)
+        SizedResults(results, sizes)
     )
 end
 function Base.getindex(br::SizedResults, row, col)
@@ -29,7 +29,7 @@ function Base.vcat(br1::BenchmarkResult, br2::BenchmarkResult)
     BenchmarkResult(
         br1.tests,
         SizedResults(
-            SharedMatrix(hcat(br1.sizedresults.results, br2.sizedresults.results)),
+            hcat(br1.sizedresults.results, br2.sizedresults.results),
             vcat(br1.sizedresults.sizes, br2.sizedresults.sizes)
         )
     )

benchmark/driver.jl

@@ -16,9 +16,6 @@ function start_workers()
     foreach(wait, map(start_worker, workers()))
 end
 stop_workers() = rmprocs(workers())
-addprocs(); nworkers()
-
-pmap_startstop(f, s) = (start_workers(); r = pmap(f, s); stop_workers(); r)
 
 blastests() = [
     "LoopVectorization",
@@ -29,110 +26,142 @@ blastests() = [
     "OpenBLAS", "MKL"
 ]    
 function benchmark_AmulB(sizes)
-    br = BenchmarkResult(blastests(), sizes)
-    sm = br.sizedresults.results
-    pmap_startstop(is -> A_mul_B_bench!(sm, is[2], is[1]), enumerate(sizes))
+    start_workers()
+    sm = SharedMatrix(Matrix{Float64}(undef, length(tests), length(sizes)))
+    pmap(is -> A_mul_B_bench!(sm, is[2], is[1]), enumerate(sizes))
+    br = BenchmarkResult(Matrix(sm), blastests(), sizes)
+    stop_workers()
     br
 end
 function benchmark_AmulBt(sizes)
-    br = BenchmarkResult(blastests(), sizes)
-    sm = br.sizedresults.results
-    pmap_startstop(is -> A_mul_Bt_bench!(sm, is[2], is[1]), enumerate(sizes))
+    start_workers()
+    sm = SharedMatrix(Matrix{Float64}(undef, length(tests), length(sizes)))
+    pmap(is -> A_mul_Bt_bench!(sm, is[2], is[1]), enumerate(sizes))
+    br = BenchmarkResult(Matrix(sm), blastests(), sizes)
+    stop_workers()
     br
 end
 function benchmark_AtmulB(sizes)
-    br = BenchmarkResult(blastests(), sizes)
-    sm = br.sizedresults.results
-    pmap_startstop(is -> At_mul_B_bench!(sm, is[2], is[1]), enumerate(sizes))
+    start_workers()
+    sm = SharedMatrix(Matrix{Float64}(undef, length(tests), length(sizes)))
+    pmap(is -> At_mul_B_bench!(sm, is[2], is[1]), enumerate(sizes))
+    br = BenchmarkResult(Matrix(sm), blastests(), sizes)
+    stop_workers()
     br
 end
 function benchmark_AtmulBt(sizes)
-    br = BenchmarkResult(blastests(), sizes)
-    sm = br.sizedresults.results
-    pmap_startstop(is -> At_mul_Bt_bench!(sm, is[2], is[1]), enumerate(sizes))
+    start_workers()
+    sm = SharedMatrix(Matrix{Float64}(undef, length(tests), length(sizes)))
+    pmap(is -> At_mul_Bt_bench!(sm, is[2], is[1]), enumerate(sizes))
+    br = BenchmarkResult(Matrix(sm), blastests(), sizes)
+    stop_workers()
     br
 end
 function benchmark_dot(sizes)
     tests = ["LoopVectorization", "Julia", "Clang", "GFortran", "icc", "ifort", "g++ & Eigen-3", "clang++ & Eigen-3"]#, "OpenBLAS"]
-    br = BenchmarkResult(tests, sizes)
-    sm = br.sizedresults.results
-    pmap_startstop(is -> dot_bench!(sm, is[2], is[1]), enumerate(sizes))
+    start_workers()
+    sm = SharedMatrix(Matrix{Float64}(undef, length(tests), length(sizes)))
+    pmap(is -> dot_bench!(sm, is[2], is[1]), enumerate(sizes))
+    br = BenchmarkResult(Matrix(sm), tests, sizes)
+    stop_workers()
     br
 end
 function benchmark_selfdot(sizes)
     tests = ["LoopVectorization", "Julia", "Clang", "GFortran", "icc", "ifort", "g++ & Eigen-3", "clang++ & Eigen-3"]#, "OpenBLAS"]
-    br = BenchmarkResult(tests, sizes)
-    sm = br.sizedresults.results
-    pmap_startstop(is -> selfdot_bench!(sm, is[2], is[1]), enumerate(sizes))
+    start_workers()
+    sm = SharedMatrix(Matrix{Float64}(undef, length(tests), length(sizes)))
+    pmap(is -> selfdot_bench!(sm, is[2], is[1]), enumerate(sizes))
+    br = BenchmarkResult(Matrix(sm), tests, sizes)
+    stop_workers()
     br
 end
 function benchmark_Amulvb(sizes)
-    br = BenchmarkResult(blastests(), sizes)
-    sm = br.sizedresults.results
-    pmap_startstop(is -> A_mul_vb_bench!(sm, is[2], is[1]), enumerate(sizes))
+    start_workers()
+    sm = SharedMatrix(Matrix{Float64}(undef, length(tests), length(sizes)))
+    pmap(is -> A_mul_vb_bench!(sm, is[2], is[1]), enumerate(sizes))
+    br = BenchmarkResult(Matrix(sm), blastests(), sizes)
+    stop_workers()
     br
 end
 function benchmark_Atmulvb(sizes)
-    br = BenchmarkResult(blastests(), sizes)
-    sm = br.sizedresults.results
-    pmap_startstop(is -> At_mul_vb_bench!(sm, is[2], is[1]), enumerate(sizes))
+    start_workers()
+    sm = SharedMatrix(Matrix{Float64}(undef, length(tests), length(sizes)))
+    pmap(is -> At_mul_vb_bench!(sm, is[2], is[1]), enumerate(sizes))
+    br = BenchmarkResult(Matrix(sm), blastests(), sizes)
+    stop_workers()
     br
 end
 function benchmark_dot3(sizes)
     tests = ["LoopVectorization", "Julia", "Clang", "GFortran", "icc", "ifort", "g++ & Eigen-3", "clang++ & Eigen-3", "LinearAlgebra" ]
-    br = BenchmarkResult(tests, sizes)
-    sm = br.sizedresults.results
-    pmap_startstop(is -> dot3_bench!(sm, is[2], is[1]), enumerate(sizes))
+    start_workers()
+    sm = SharedMatrix(Matrix{Float64}(undef, length(tests), length(sizes)))
+    pmap(is -> dot3_bench!(sm, is[2], is[1]), enumerate(sizes))
+    br = BenchmarkResult(Matrix(sm), tests, sizes)
+    stop_workers()
     br
 end
 function benchmark_sse(sizes)
     tests = ["LoopVectorization", "Julia", "Clang", "GFortran", "icc", "ifort", "g++ & Eigen-3", "clang++ & Eigen-3", "MKL"]
-    br = BenchmarkResult(tests, sizes)
-    sm = br.sizedresults.results
-    pmap_startstop(is -> sse_bench!(sm, is[2], is[1]), enumerate(sizes))
+    start_workers()
+    sm = SharedMatrix(Matrix{Float64}(undef, length(tests), length(sizes)))
+    pmap(is -> sse_bench!(sm, is[2], is[1]), enumerate(sizes))
+    br = BenchmarkResult(Matrix(sm), tests, sizes)
+    stop_workers()
     br
 end
 function benchmark_exp(sizes)
     tests = ["LoopVectorization", "Julia", "Clang", "GFortran", "icc", "ifort"]
-    br = BenchmarkResult(tests, sizes)
-    sm = br.sizedresults.results
-    pmap_startstop(is -> exp_bench!(sm, is[2], is[1]), enumerate(sizes))
+    start_workers()
+    sm = SharedMatrix(Matrix{Float64}(undef, length(tests), length(sizes)))
+    pmap(is -> exp_bench!(sm, is[2], is[1]), enumerate(sizes))
+    br = BenchmarkResult(Matrix(sm), tests, sizes)
+    stop_workers()
     br
 end
 function benchmark_aplusBc(sizes)
     tests = ["LoopVectorization", "Julia", "Clang", "GFortran", "icc", "ifort", "g++ & Eigen-3", "clang++ & Eigen-3"]
-    br = BenchmarkResult(tests, sizes)
-    sm = br.sizedresults.results
-    pmap_startstop(is -> aplusBc_bench!(sm, is[2], is[1]), enumerate(sizes))
+    start_workers()
+    sm = SharedMatrix(Matrix{Float64}(undef, length(tests), length(sizes)))
+    pmap(is -> aplusBc_bench!(sm, is[2], is[1]), enumerate(sizes))
+    br = BenchmarkResult(Matrix(sm), tests, sizes)
+    stop_workers()
     br
 end
 function benchmark_AplusAt(sizes)
     tests = ["LoopVectorization", "Julia", "Clang", "GFortran", "icc", "ifort", "g++ & Eigen-3", "clang++ & Eigen-3", "GFortran-builtin", "ifort-builtin"]
-    br = BenchmarkResult(tests, sizes)
-    sm = br.sizedresults.results
-    pmap_startstop(is -> AplusAt_bench!(sm, is[2], is[1]), enumerate(sizes))
+    start_workers()
+    sm = SharedMatrix(Matrix{Float64}(undef, length(tests), length(sizes)))
+    pmap(is -> AplusAt_bench!(sm, is[2], is[1]), enumerate(sizes))
+    br = BenchmarkResult(Matrix(sm), tests, sizes)
+    stop_workers()
     br
 end
 function benchmark_random_access(sizes)
     tests = ["LoopVectorization", "Julia", "Clang", "GFortran", "icc", "ifort"]
-    br = BenchmarkResult(tests, sizes)
-    sm = br.sizedresults.results
-    pmap_startstop(is -> randomaccess_bench!(sm, is[2], is[1]), enumerate(sizes))
+    start_workers()
+    sm = SharedMatrix(Matrix{Float64}(undef, length(tests), length(sizes)))
+    pmap(is -> randomaccess_bench!(sm, is[2], is[1]), enumerate(sizes))
+    br = BenchmarkResult(Matrix(sm), tests, sizes)
+    stop_workers()
     br
 end
 function benchmark_logdettriangle(sizes)
     # tests = ["LoopVectorization", "Julia", "Clang", "GFortran", "icc", "ifort", "g++ & Eigen-3", "clang++ & Eigen-3", "LinearAlgebra"]
     tests = ["LoopVectorization", "Julia", "Clang", "GFortran", "icc", "ifort", "LinearAlgebra"]
-    br = BenchmarkResult(tests, sizes)
-    sm = br.sizedresults.results
-    pmap_startstop(is -> logdettriangle_bench!(sm, is[2], is[1]), enumerate(sizes))
+    start_workers()
+    sm = SharedMatrix(Matrix{Float64}(undef, length(tests), length(sizes)))
+    pmap(is -> logdettriangle_bench!(sm, is[2], is[1]), enumerate(sizes))
+    br = BenchmarkResult(Matrix(sm), tests, sizes)
+    stop_workers()
     br
 end
 function benchmark_filter2d(sizes, K)
     tests = ["LoopVectorization", "Julia", "Clang", "GFortran", "icc", "ifort"]
-    br = BenchmarkResult(tests, sizes)
-    sm = br.sizedresults.results
-    pmap_startstop(is -> filter2d_bench_run!(sm, is[2], is[1], K), enumerate(sizes))
+    start_workers()
+    sm = SharedMatrix(Matrix{Float64}(undef, length(tests), length(sizes)))
+    pmap(is -> filter2d_bench_run!(sm, is[2], is[1], K), enumerate(sizes))
+    br = BenchmarkResult(Matrix(sm), tests, sizes)
+    stop_workers()
     br
 end
 function benchmark_filter2ddynamic(sizes)
@@ -145,10 +174,12 @@ function benchmark_filter2d3x3(sizes)
 end
 function benchmark_filter2dunrolled(sizes)
     tests = ["LoopVectorization", "Julia", "Clang", "GFortran", "icc", "ifort"]
-    br = BenchmarkResult(tests, sizes)
-    sm = br.sizedresults.results
+    start_workers()
+    sm = SharedMatrix(Matrix{Float64}(undef, length(tests), length(sizes)))
     K = SizedOffsetMatrix{Float64,-1,1,-1,1}(rand(3,3))
-    pmap_startstop(is -> filter2dunrolled_bench_run!(sm, is[2], is[1], K), enumerate(sizes))
+    pmap(is -> filter2dunrolled_bench_run!(sm, is[2], is[1], K), enumerate(sizes))
+    br = BenchmarkResult(Matrix(sm), tests, sizes)
+    stop_workers()
     br
 end
 
@@ -171,6 +202,7 @@ Atmulvb_bench = benchmark_Atmulvb(sizes); println("A' * b benchmark results:");
 
 dot_bench = benchmark_dot(longsizes); println("a' * b benchmark results:"); println(dot_bench)
 selfdot_bench = benchmark_selfdot(longsizes); println("a' * a benchmark results:"); println(selfdot_bench)
+
 sse_bench = benchmark_sse(sizes); println("Benchmark resutls of summing squared error:"); println(sse_bench)
 aplusBc_bench = benchmark_aplusBc(sizes); println("Benchmark results of a .+ B .* c':"); println(aplusBc_bench)
 AplusAt_bench = benchmark_AplusAt(sizes); println("Benchmark results of A * A':"); println(AplusAt_bench)
@@ -187,6 +219,10 @@ using Cairo, Fontconfig
 const PICTURES = joinpath(pkgdir(LoopVectorization), "docs", "src", "assets")
 saveplot(f, br) = draw(PNG(joinpath(PICTURES, f * "$v.png"), 12inch, 8inch), plot(br))
 
+# If only rerunning a few, remove them from load.
+# @load "benchmarkresults.jld2" logdettriangle_bench filter2d_dynamic_bench filter2d_3x3_bench filter2d_unrolled_bench dot_bench selfdot_bench dot3_bench sse_bench aplusBc_bench AplusAt_bench vexp_bench randomaccess_bench AmulB_bench AmulBt_bench AtmulB_bench AtmulBt_bench Amulvb_bench Atmulvb_bench
+@load "benchmarkresults.jld2" logdettriangle_bench filter2d_dynamic_bench filter2d_3x3_bench filter2d_unrolled_bench dot3_bench sse_bench aplusBc_bench AplusAt_bench vexp_bench randomaccess_bench AmulB_bench AmulBt_bench AtmulB_bench AtmulBt_bench Amulvb_bench Atmulvb_bench
+
 saveplot("bench_dot3_v", dot3_bench);
 saveplot("bench_dot_v", dot_bench);
 saveplot("bench_selfdot_v", selfdot_bench);
@@ -207,7 +243,7 @@ saveplot("bench_filter2d_unrolled_v", filter2d_unrolled_bench);
 saveplot("bench_exp_v", vexp_bench);
 saveplot("bench_random_access_v", randomaccess_bench);
 
-# @load "benchmarkresults.jld2" logdettriangle_bench filter2d_dynamic_bench filter2d_3x3_bench filter2d_unrolled_bench vexp_bench randomaccess_bench
+
 
 @save "benchmarkresults.jld2" logdettriangle_bench filter2d_dynamic_bench filter2d_3x3_bench filter2d_unrolled_bench dot_bench selfdot_bench dot3_bench sse_bench aplusBc_bench AplusAt_bench vexp_bench randomaccess_bench AmulB_bench AmulBt_bench AtmulB_bench AtmulBt_bench Amulvb_bench Atmulvb_bench