Julia script for benchmarking on top of current setup

Author: shravanngoswamii

benchmarks/Project.toml

@@ -11,6 +11,7 @@ Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 DrWatson = "634d3b9d-ee7a-5ddf-bec9-22491ea816e1"
 DynamicPPL = "366bfd00-2699-11ea-058f-f148b4cae6d8"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
 Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

benchmarks/benchmarks.jl

@@ -0,0 +1,94 @@
+using BenchmarkTools
+using DynamicPPL
+using Distributions
+using DynamicPPLBenchmarks: time_model_def, make_suite
+using PrettyTables
+using Dates
+using LibGit2
+
+const RESULTS_DIR = "results"
+const BENCHMARK_NAME = let
+    repo = try
+        LibGit2.GitRepo(joinpath(pkgdir(DynamicPPL), ".."))
+    catch
+        nothing
+    end
+    isnothing(repo) ? "benchmarks_$(Dates.format(now(), "yyyy-mm-dd_HH-MM-SS"))" :
+    "$(LibGit2.headname(repo))_$(string(LibGit2.GitHash(LibGit2.peel(LibGit2.GitCommit, LibGit2.head(repo))))[1:6])"
+end
+
+mkpath(joinpath(RESULTS_DIR, BENCHMARK_NAME))
+
+@model function demo1(x)
+    m ~ Normal()
+    x ~ Normal(m, 1)
+    return (m = m, x = x)
+end
+
+@model function demo2(y)
+    p ~ Beta(1, 1)
+    N = length(y)
+    for n in 1:N
+        y[n] ~ Bernoulli(p)
+    end
+    return (; p)
+end
+
+models = [
+    (name = "demo1", model = demo1, data = (1.0,)),
+    (name = "demo2", model = demo2, data = (rand(0:1, 10),))
+]
+
+results = []
+for (model_name, model_def, data) in models
+    println(">> Running benchmarks for model: $model_name")
+    m = time_model_def(model_def, data...)
+    println()
+    suite = make_suite(m)
+    bench_results = run(suite, seconds=10)
+    
+    output_path = joinpath(RESULTS_DIR, BENCHMARK_NAME, "$(model_name)_benchmarks.json")
+    BenchmarkTools.save(output_path, bench_results)
+    
+    for (eval_type, trial) in bench_results
+        push!(results, (
+            Model = model_name,
+            Evaluation = eval_type,
+            Time = minimum(trial).time,
+            Memory = trial.memory,
+            Allocations = trial.allocs,
+            Samples = length(trial.times)
+        ))
+    end
+end
+
+formatted = map(results) do r
+    (Model = r.Model,
+     Evaluation = replace(r.Evaluation, "_" => " "),
+     Time = BenchmarkTools.prettytime(r.Time),
+     Memory = BenchmarkTools.prettymemory(r.Memory),
+     Allocations = string(r.Allocations),
+     Samples = string(r.Samples))
+end
+
+md_output = """
+## DynamicPPL Benchmark Results ($BENCHMARK_NAME)
+
+### Execution Environment
+- Julia version: $(VERSION)
+- DynamicPPL version: $(pkgversion(DynamicPPL))
+- Benchmark date: $(now())
+
+$(pretty_table(String, formatted,
+    tf = tf_markdown,
+    header = ["Model", "Evaluation Type", "Time", "Memory", "Allocs", "Samples"],
+    alignment = [:l, :l, :r, :r, :r, :r]
+))
+"""
+
+println(md_output)
+open(joinpath(RESULTS_DIR, BENCHMARK_NAME, "REPORT.md"), "w") do io
+    write(io, md_output)
+end
+
+println("Benchmark results saved to: $RESULTS_DIR/$BENCHMARK_NAME")

benchmarks/results/benchmarks_2025-02-02_04-21-46/REPORT.md

@@ -0,0 +1,22 @@
+## DynamicPPL Benchmark Results (benchmarks_2025-02-02_04-21-46)
+
+### Execution Environment
+- Julia version: 1.10.5
+- DynamicPPL version: 0.32.2
+- Benchmark date: 2025-02-02T04:22:00.341
+
+| Model | Evaluation Type                           |       Time |    Memory | Allocs | Samples |
+|-------|-------------------------------------------|------------|-----------|--------|---------|
+| demo1 | evaluation typed                          | 194.000 ns | 160 bytes |      3 |   10000 |
+| demo1 | evaluation untyped                        |   1.027 μs |  1.52 KiB |     32 |   10000 |
+| demo1 | evaluation simple varinfo dict            | 694.000 ns | 704 bytes |     26 |   10000 |
+| demo1 | evaluation simple varinfo nt              |  43.000 ns |   0 bytes |      0 |   10000 |
+| demo1 | evaluation simple varinfo dict from nt    |  48.000 ns |   0 bytes |      0 |   10000 |
+| demo1 | evaluation simple varinfo componentarrays |  44.000 ns |   0 bytes |      0 |   10000 |
+| demo2 | evaluation typed                          | 273.000 ns | 160 bytes |      3 |   10000 |
+| demo2 | evaluation untyped                        |   2.528 μs |  3.47 KiB |     67 |   10000 |
+| demo2 | evaluation simple varinfo dict            |   2.189 μs |  1.42 KiB |     60 |   10000 |
+| demo2 | evaluation simple varinfo nt              | 136.000 ns |   0 bytes |      0 |   10000 |
+| demo2 | evaluation simple varinfo dict from nt    | 119.000 ns |   0 bytes |      0 |   10000 |
+| demo2 | evaluation simple varinfo componentarrays | 136.000 ns |   0 bytes |      0 |   10000 |
+