|
| 1 | +module DynamicPPLBenchmarks |
| 2 | + |
| 3 | +using DynamicPPL |
| 4 | +using BenchmarkTools |
| 5 | + |
| 6 | +using Weave: Weave |
| 7 | +using Markdown: Markdown |
| 8 | + |
| 9 | +using LibGit2: LibGit2 |
| 10 | +using Pkg: Pkg |
| 11 | + |
| 12 | +export weave_benchmarks |
| 13 | + |
| 14 | +function time_model_def(model_def, args...) |
| 15 | + return @time model_def(args...) |
| 16 | +end |
| 17 | + |
| 18 | +function benchmark_untyped_varinfo!(suite, m) |
| 19 | + vi = VarInfo() |
| 20 | + # Populate. |
| 21 | + m(vi) |
| 22 | + # Evaluate. |
| 23 | + suite["evaluation_untyped"] = @benchmarkable $m($vi, $(DefaultContext())) |
| 24 | + return suite |
| 25 | +end |
| 26 | + |
| 27 | +function benchmark_typed_varinfo!(suite, m) |
| 28 | + # Populate. |
| 29 | + vi = VarInfo(m) |
| 30 | + # Evaluate. |
| 31 | + suite["evaluation_typed"] = @benchmarkable $m($vi, $(DefaultContext())) |
| 32 | + return suite |
| 33 | +end |
| 34 | + |
| 35 | +function typed_code(m, vi=VarInfo(m)) |
| 36 | + rng = DynamicPPL.Random.MersenneTwister(42) |
| 37 | + spl = DynamicPPL.SampleFromPrior() |
| 38 | + ctx = DynamicPPL.SamplingContext(rng, spl, DynamicPPL.DefaultContext()) |
| 39 | + |
| 40 | + results = code_typed(m.f, Base.typesof(m, vi, ctx, m.args...)) |
| 41 | + return first(results) |
| 42 | +end |
| 43 | + |
| 44 | +""" |
| 45 | + make_suite(model) |
| 46 | +
|
| 47 | +Create default benchmark suite for `model`. |
| 48 | +""" |
| 49 | +function make_suite(model) |
| 50 | + suite = BenchmarkGroup() |
| 51 | + benchmark_untyped_varinfo!(suite, model) |
| 52 | + benchmark_typed_varinfo!(suite, model) |
| 53 | + |
| 54 | + return suite |
| 55 | +end |
| 56 | + |
| 57 | +""" |
| 58 | + weave_child(indoc; mod, args, kwargs...) |
| 59 | +
|
| 60 | +Weave `indoc` with scope of `mod` into markdown. |
| 61 | +
|
| 62 | +Useful for weaving within weaving, e.g. |
| 63 | +```julia |
| 64 | +weave_child(child_jmd_path, mod = @__MODULE__, args = WEAVE_ARGS) |
| 65 | +``` |
| 66 | +together with `results="markup"` and `echo=false` will simply insert |
| 67 | +the weaved version of `indoc`. |
| 68 | +
|
| 69 | +# Notes |
| 70 | +- Currently only supports `doctype == "github"`. Other outputs are "supported" |
| 71 | + in the sense that it works but you might lose niceties such as syntax highlighting. |
| 72 | +""" |
| 73 | +function weave_child(indoc; mod, args, kwargs...) |
| 74 | + # FIXME: Make this work for other output formats than just `github`. |
| 75 | + doc = Weave.WeaveDoc(indoc, nothing) |
| 76 | + doc = Weave.run_doc(doc; doctype="github", mod=mod, args=args, kwargs...) |
| 77 | + rendered = Weave.render_doc(doc) |
| 78 | + return display(Markdown.parse(rendered)) |
| 79 | +end |
| 80 | + |
| 81 | +""" |
| 82 | + pkgversion(m::Module) |
| 83 | +
|
| 84 | +Return version of module `m` as listed in its Project.toml. |
| 85 | +""" |
| 86 | +function pkgversion(m::Module) |
| 87 | + projecttoml_path = joinpath(dirname(pathof(m)), "..", "Project.toml") |
| 88 | + return Pkg.TOML.parsefile(projecttoml_path)["version"] |
| 89 | +end |
| 90 | + |
| 91 | +""" |
| 92 | + default_name(; include_commit_id=false) |
| 93 | +
|
| 94 | +Construct a name from either repo information or package version |
| 95 | +of `DynamicPPL`. |
| 96 | +
|
| 97 | +If the path of `DynamicPPL` is a git-repo, return name of current branch, |
| 98 | +joined with the commit id if `include_commit_id` is `true`. |
| 99 | +
|
| 100 | +If path of `DynamicPPL` is _not_ a git-repo, it is assumed to be a release, |
| 101 | +resulting in a name of the form `release-VERSION`. |
| 102 | +""" |
| 103 | +function default_name(; include_commit_id=false) |
| 104 | + dppl_path = abspath(joinpath(dirname(pathof(DynamicPPL)), "..")) |
| 105 | + |
| 106 | + # Extract branch name and commit id |
| 107 | + local name |
| 108 | + try |
| 109 | + githead = LibGit2.head(LibGit2.GitRepo(dppl_path)) |
| 110 | + branchname = LibGit2.shortname(githead) |
| 111 | + |
| 112 | + name = replace(branchname, "/" => "_") |
| 113 | + if include_commit_id |
| 114 | + gitcommit = LibGit2.peel(LibGit2.GitCommit, githead) |
| 115 | + commitid = string(LibGit2.GitHash(gitcommit)) |
| 116 | + name *= "-$(commitid)" |
| 117 | + end |
| 118 | + catch e |
| 119 | + if e isa LibGit2.GitError |
| 120 | + @info "No git repo found for $(dppl_path); extracting name from package version." |
| 121 | + name = "release-$(pkgversion(DynamicPPL))" |
| 122 | + else |
| 123 | + rethrow(e) |
| 124 | + end |
| 125 | + end |
| 126 | + |
| 127 | + return name |
| 128 | +end |
| 129 | + |
| 130 | +""" |
| 131 | + weave_benchmarks(input="benchmarks.jmd"; kwargs...) |
| 132 | +
|
| 133 | +Weave benchmarks present in `benchmarks.jmd` into a single file. |
| 134 | +
|
| 135 | +# Keyword arguments |
| 136 | +- `benchmarkbody`: JMD-file to be rendered for each model. |
| 137 | +- `include_commit_id=false`: specify whether to include commit-id in the default name. |
| 138 | +- `name`: the name of directory in `results/` to use as output directory. |
| 139 | +- `name_old=nothing`: if specified, comparisons of current run vs. the run pinted to |
| 140 | + by `name_old` will be included in the generated document. |
| 141 | +- `include_typed_code=false`: if `true`, output of `code_typed` for the evaluator |
| 142 | + of the model will be included in the weaved document. |
| 143 | +- Rest of the passed `kwargs` will be passed on to `Weave.weave`. |
| 144 | +""" |
| 145 | +function weave_benchmarks( |
| 146 | + input=joinpath(dirname(pathof(DynamicPPLBenchmarks)), "..", "benchmarks.jmd"); |
| 147 | + benchmarkbody=joinpath( |
| 148 | + dirname(pathof(DynamicPPLBenchmarks)), "..", "benchmark_body.jmd" |
| 149 | + ), |
| 150 | + include_commit_id=false, |
| 151 | + name=default_name(; include_commit_id=include_commit_id), |
| 152 | + name_old=nothing, |
| 153 | + include_typed_code=false, |
| 154 | + doctype="github", |
| 155 | + outpath="results/$(name)/", |
| 156 | + kwargs..., |
| 157 | +) |
| 158 | + args = Dict( |
| 159 | + :benchmarkbody => benchmarkbody, |
| 160 | + :name => name, |
| 161 | + :include_typed_code => include_typed_code, |
| 162 | + ) |
| 163 | + if !isnothing(name_old) |
| 164 | + args[:name_old] = name_old |
| 165 | + end |
| 166 | + @info "Storing output in $(outpath)" |
| 167 | + mkpath(outpath) |
| 168 | + return Weave.weave(input, doctype; out_path=outpath, args=args, kwargs...) |
| 169 | +end |
| 170 | + |
| 171 | +end # module |
![github-actions[bot]](https://avatars.githubusercontent.com/in/15368?v=4&size=40){kind=avatar}
0 commit comments