-
Notifications
You must be signed in to change notification settings - Fork 72
Add benchmarks #738
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add benchmarks #738
Changes from all commits
a101c10
8d54daf
7bdbdf4
69e7e95
1816837
8518bfe
fef1c88
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| name: Benchmark this PR | ||
| on: | ||
| pull_request_target: | ||
| branches: [ master ] # change to your default branch | ||
| permissions: | ||
| pull-requests: write # needed to post comments | ||
|
|
||
| jobs: | ||
| bench: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: MilesCranmer/AirspeedVelocity.jl@action-v1 | ||
| with: | ||
| julia-version: '1' |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| [deps] | ||
| BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" | ||
| IntervalArithmetic = "d1acc4aa-44c8-5952-acd4-ba5d80a2a253" | ||
| MPFI = "e50a78b8-6e2f-482e-b7c3-776aa2235d1a" | ||
| Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" | ||
|
|
||
| [sources] | ||
| MPFI = {url = "https://gitlab.inria.fr/ckatsama/mpfi.jl.git"} |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| using BenchmarkTools | ||
| using CairoMakie | ||
| using DataFrames | ||
|
|
||
| include("benchmarks.jl") | ||
|
|
||
| results = run(SUITE ; verbose = true) | ||
|
|
||
| df = DataFrame(; constructor = String[], suite = String[], f = String[], trial = BenchmarkTools.Trial[]) | ||
|
|
||
| for (name, T) in interval_constructors | ||
| # for suite in suites | ||
| begin | ||
| suite = "basics" | ||
| suite_df = DataFrame(results[name][suite], [:f, :trial]) | ||
| suite_df[:, :constructor] .= name | ||
| suite_df[:, :suite] .= suite | ||
| df = vcat(df, suite_df) | ||
| end | ||
| end | ||
|
|
||
| transform!(df, | ||
| :trial => ByRow(trial -> minimum(trial.times)) => :minimum, | ||
| :trial => ByRow(trial -> mean(trial.times)) => :mean, | ||
| :trial => ByRow(trial -> median(trial.times)) => :median | ||
| ) | ||
|
|
||
| df[:, :relative] .= 0.0 | ||
|
|
||
| for group in groupby(df, :f) | ||
| group[:, :relative] .= group[:, :median] ./ only(group[group.constructor .== "bareinterval", :median]) | ||
| end | ||
|
|
||
| begin | ||
| fig = Figure(size = (800, 500)) | ||
|
|
||
| data = df[df.suite .== "basics", :] | ||
|
|
||
| fs = vcat(string.(basic_arithmetic), string.(basic_functions)) | ||
| to_x = Dict(f => k for (k, f) in enumerate(fs)) | ||
| xx = [to_x[f] for f in data.f] | ||
|
|
||
| constructors = ["bareinterval", "interval", "BigFloat bareinterval", "BigFloat interval", "BigFloat MPFI"] | ||
| to_dodge = Dict(constructor => k for (k, constructor) in enumerate(constructors)) | ||
| dodge = [to_dodge[constructor] for constructor in data.constructor] | ||
|
|
||
| ax = Axis(fig[1, 1] ; | ||
| xlabel = "Benchmarked function", | ||
| xticks = (1:length(fs), fs), | ||
| ylabel = "Relative execution time (log scale)", | ||
| yscale = log10, | ||
| ) | ||
|
|
||
| barplot!(ax, xx, data.relative ; | ||
| dodge, | ||
| color = dodge, | ||
| colormap = :mpetroff_10, | ||
| colorrange = (1, 10), | ||
| ) | ||
|
|
||
| Legend(fig[0, 1], | ||
| [LineElement(linewidth = 20, color = to_colormap(:mpetroff_10)[k]) for k in eachindex(constructors)], | ||
| constructors ; | ||
| tellwidth = false, | ||
| orientation = :horizontal | ||
| ) | ||
|
|
||
| fig | ||
| end |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| using BenchmarkTools | ||
| using IntervalArithmetic | ||
|
|
||
| import MPFI | ||
| import Random | ||
|
|
||
| MPFI.BigInterval(x::ExactReal) = MPFI.BigInterval(x.value) | ||
| Base.:(^)(x::MPFI.BigInterval, y::ExactReal) = x^y.value | ||
|
|
||
| Random.seed!(0) | ||
|
|
||
| basic_arithmetic = [+, *, -, /] | ||
| basic_functions = [exp, cosh, sinh, tanh, inv, sqrt, abs, log, sin, cos, tan, acos, asin, atan] | ||
|
|
||
| interval_constructors = Dict( | ||
| "bareinterval" => bareinterval, | ||
| "interval" => interval, | ||
| "BigFloat bareinterval" => (x, y) -> bareinterval(BigFloat, x, y), | ||
| "BigFloat interval" => (x, y) -> interval(BigFloat, x, y), | ||
| "BigFloat MPFI" => MPFI.BigInterval | ||
| ) | ||
|
|
||
| suites = ["basics", "Dietmar-Ratz"] | ||
|
|
||
| bounds = map(range(0.01, 10 ; length = 100)) do i | ||
| x = i * randn() | ||
| y = i * randn() | ||
| x > y && return (y, x) | ||
| return (x, y) | ||
| end | ||
|
|
||
| SUITE = BenchmarkGroup() | ||
|
|
||
| for (name, T) in interval_constructors | ||
| xx = [T(x, y) for (x, y) in bounds] | ||
| yy = reverse(xx) | ||
|
|
||
| SUITE[name] = BenchmarkGroup(split(name)) | ||
| SUITE[name]["basics"] = BenchmarkGroup(["arithmetic"]) | ||
| SUITE[name]["Dietmar-Ratz"] = BenchmarkGroup(["arithmetic"]) | ||
|
|
||
| for f in basic_functions | ||
| SUITE[name]["basics"][string(f)] = @benchmarkable ($f).($xx) | ||
| end | ||
|
|
||
| for op in basic_arithmetic | ||
| SUITE[name]["basics"][string(op)] = @benchmarkable ($op).($xx, $yy) | ||
| end | ||
| end |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -14,7 +14,12 @@ Implement the `inf` function of the IEEE Standard 1788-2015 (Table 9.2). | |
| See also: [`sup`](@ref), [`bounds`](@ref), [`mid`](@ref), [`diam`](@ref), | ||
| [`radius`](@ref) and [`midradius`](@ref). | ||
| """ | ||
| inf(x::BareInterval{T}) where {T<:AbstractFloat} = ifelse(isnan(x.lo), typemax(T), ifelse(iszero(x.lo), copysign(x.lo, -1), x.lo)) | ||
| function inf(x::BareInterval{T}) where {T<:AbstractFloat} | ||
| isnan(x.lo) && return typemax(T) | ||
| iszero(x.lo) && return copysign(x.lo, -1) | ||
| return x.lo | ||
| end | ||
|
|
||
| inf(x::BareInterval{<:Rational}) = x.lo | ||
|
|
||
| function inf(x::Interval{T}) where {T<:AbstractFloat} | ||
|
|
@@ -38,7 +43,11 @@ Implement the `sup` function of the IEEE Standard 1788-2015 (Table 9.2). | |
| See also: [`inf`](@ref), [`bounds`](@ref), [`mid`](@ref), [`diam`](@ref), | ||
| [`radius`](@ref) and [`midradius`](@ref). | ||
| """ | ||
| sup(x::BareInterval{T}) where {T<:AbstractFloat} = ifelse(isnan(x.hi), typemin(T), x.hi) | ||
| function sup(x::BareInterval{T}) where {T<:AbstractFloat} | ||
| isnan(x.hi) && return typemin(T) | ||
| return x.hi | ||
| end | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Specialise to
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Are there reason to believe that in a larger code the branching would make a significant difference? Looking at only microbenchmarks, both versions seem to be equivalent for Float64: julia> s1(x) = ifelse(isnan(x.hi), typemin(x.hi), x.hi)
s1 (generic function with 1 method)
julia> function s2(x)
isnan(x.hi) && return typemin(x.hi)
return x.hi
end
s2 (generic function with 1 method)
julia> @benchmark s1(X) setup=(X = bareinterval(rand(), rand() + 1))
BenchmarkTools.Trial: 10000 samples with 1000 evaluations per sample.
Range (min … max): 2.654 ns … 24.137 ns ┊ GC (min … max): 0.00% … 0.00%
Time (median): 2.805 ns ┊ GC (median): 0.00%
Time (mean ± σ): 2.822 ns ± 0.711 ns ┊ GC (mean ± σ): 0.00% ± 0.00%
█ ▃
█▁▁▁▁█▄▂▅▂▁▂▂▂▂▂▂▁▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂ ▂
2.65 ns Histogram: frequency by time 4.2 ns <
Memory estimate: 0 bytes, allocs estimate: 0.
julia> @benchmark s2(X) setup=(X = bareinterval(rand(), rand() + 1))
BenchmarkTools.Trial: 10000 samples with 1000 evaluations per sample.
Range (min … max): 2.669 ns … 22.299 ns ┊ GC (min … max): 0.00% … 0.00%
Time (median): 2.806 ns ┊ GC (median): 0.00%
Time (mean ± σ): 2.829 ns ± 0.550 ns ┊ GC (mean ± σ): 0.00% ± 0.00%
▅ ▇█
█▂▁▁▁██▂▂▂▁▁▁▁▂▂▂▂▁▁▂▁▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂ ▂
2.67 ns Histogram: frequency by time 4 ns <
Memory estimate: 0 bytes, allocs estimate: 0.
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. That being said in this benchmark, all samples are taking the same branch, it may bias the benchmark.
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes; though I am not sure I'd say "significant". From my understanding it is a "free" micro-optimisation (which the compiler might end up doing anyways) when the |
||
|
|
||
| sup(x::BareInterval{<:Rational}) = x.hi | ||
|
|
||
| function sup(x::Interval{T}) where {T<:AbstractFloat} | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Specialise to
T<:BigFloatand keep the branch free version