new testproblems

Author: epolack

benchmark/Project.toml

@@ -1,6 +1,6 @@
 [deps]
+AtomsIOPython = "9e4c859b-2281-48ef-8059-f50fe53c37b0"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
-Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d"
 DFTK = "acf6eb54-70d9-11e9-0013-234b7a5f5337"
 LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

benchmark/ase.jl

@@ -1,19 +1,10 @@
 # https://opensourc.es/blog/constraint-solver-bugs-and-benchmarks/
 # https://github.com/JuliaSymbolics/Symbolics.jl/blob/master/benchmark/benchmarks.jl
 using BenchmarkTools
-import LibGit2
-
-#import Pkg; Pkg.add("Conda")
-#using Conda
-#pkgs = ["ase", "pymatgen"]
-#!Sys.iswindows() && (pkgs = append!(["libblas=*=*netlib", "nomkl"], pkgs))
-#Conda.add(pkgs; channel="conda-forge")
-
-include("load_from_file.jl")
-repo = mktempdir()
-LibGit2.clone("https://github.com/mfherbst/DFTK-testproblems", repo)
+using TestItemRunner
 
 const SUITE = BenchmarkGroup()
 
 SUITE["AlSiO2H"] = BenchmarkGroup()
-SUITE["AlSiO2H"] = @benchmarkable include(joinpath(repo, "AlSiO2H.jl"))
+SUITE["AlSiO2H"] = @benchmarkable @run_package_tests filter=(i->occursin("AlSiO2H.jl", i.filename) &&
+                                                             :debug ∈ i.tags)

benchmark/benchmarks.jl

@@ -8,13 +8,18 @@ end
 
 using PkgBenchmark
 
-tempdir = mktempdir()
-script = tempname(tempdir) * ".jl"
+import LibGit2
+repo = mktempdir(mktempdir())  # TestItemRunner needs access to parent directory as well.
+LibGit2.clone("https://github.com/epolack/DFTK-testproblems", repo)
+
+#script = tempname() * ".jl"
+script = tempname(repo) * ".jl"
 benchpath = joinpath(ROOTPATH, "benchmark", "benchmarks.jl")
 cp(benchpath, script)
-cp(joinpath(ROOTPATH, "benchmark", "load_from_file.jl"), joinpath(tempdir, "load_from_file.jl"))
 
-juliacmd=`$(joinpath(Sys.BINDIR, Base.julia_exename())) -O3 -e"import Pkg; Pkg.activate(\"$ROOTPATH\"); Pkg.instantiate();"`
+# Reconfigures the packages in case of version changes.
+juliacmd=`$(joinpath(Sys.BINDIR, Base.julia_exename())) \
+          -O3 -e"import Pkg; Pkg.activate(\"$ROOTPATH\"); Pkg.instantiate();"`
 current = BenchmarkConfig(; id="HEAD", juliacmd)
 baseline = BenchmarkConfig(; id="master", juliacmd)
 judged = judge(ROOTPATH, current, baseline; retune=true, script)