Skip to content

Commit 22ec656

Browse files
ChrisRackauckasChrisRackauckas
authored
Merge pull request #136 from avik-pal/ap/benchmark_ci
Run a Benchmark CI for PRs for some baseline results
2 parents 00d8e04 + 53d6751 commit 22ec656

File tree

6 files changed

+184
-0
lines changed

6 files changed

+184
-0
lines changed

‎.github/workflows/BenchmarkPR.yml

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
name: Benchmark Pull Request
2+
3+
on:
4+
pull_request_target:
5+
branches:
6+
- master
7+
8+
permissions:
9+
pull-requests: write
10+
11+
jobs:
12+
generate_plots:
13+
runs-on: ubuntu-latest
14+
15+
steps:
16+
- uses: actions/checkout@v2
17+
- uses: julia-actions/setup-julia@v1
18+
with:
19+
version: "1"
20+
- uses: julia-actions/cache@v1
21+
- name: Extract Package Name from Project.toml
22+
id: extract-package-name
23+
run: |
24+
PACKAGE_NAME=$(grep "^name" Project.toml | sed 's/^name = "\(.*\)"$/\1/')
25+
echo "::set-output name=package_name::$PACKAGE_NAME"
26+
- name: Build AirspeedVelocity
27+
env:
28+
JULIA_NUM_THREADS: 2
29+
run: |
30+
# Lightweight build step, as sometimes the runner runs out of memory:
31+
julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.add(;url="https://github.com/MilesCranmer/AirspeedVelocity.jl.git")'
32+
julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.build("AirspeedVelocity")'
33+
- name: Add ~/.julia/bin to PATH
34+
run: |
35+
echo "$HOME/.julia/bin" >> $GITHUB_PATH
36+
- name: Run benchmarks
37+
run: |
38+
echo $PATH
39+
ls -l ~/.julia/bin
40+
mkdir results
41+
benchpkg ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --url=${{ github.event.repository.clone_url }} --bench-on="${{github.event.repository.default_branch}}" --output-dir=results/ --tune
42+
- name: Create plots from benchmarks
43+
run: |
44+
mkdir -p plots
45+
benchpkgplot ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --npart=10 --format=png --input-dir=results/ --output-dir=plots/
46+
- name: Upload plot as artifact
47+
uses: actions/upload-artifact@v2
48+
with:
49+
name: plots
50+
path: plots
51+
- name: Create markdown table from benchmarks
52+
run: |
53+
benchpkgtable ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --input-dir=results/ --ratio > table.md
54+
echo '### Benchmark Results' > body.md
55+
echo '' >> body.md
56+
echo '' >> body.md
57+
cat table.md >> body.md
58+
echo '' >> body.md
59+
echo '' >> body.md
60+
echo '### Benchmark Plots' >> body.md
61+
echo 'A plot of the benchmark results have been uploaded as an artifact to the workflow run for this PR.' >> body.md
62+
echo 'Go to "Actions"->"Benchmark a pull request"->[the most recent run]->"Artifacts" (at the bottom).' >> body.md
63+
64+
- name: Find Comment
65+
uses: peter-evans/find-comment@v2
66+
id: fcbenchmark
67+
with:
68+
issue-number: ${{ github.event.pull_request.number }}
69+
comment-author: "github-actions[bot]"
70+
body-includes: Benchmark Results
71+
72+
- name: Comment on PR
73+
uses: peter-evans/create-or-update-comment@v3
74+
with:
75+
comment-id: ${{ steps.fcbenchmark.outputs.comment-id }}
76+
issue-number: ${{ github.event.pull_request.number }}
77+
body-path: body.md
78+
edit-mode: replace

‎.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,7 @@ Manifest.toml
55
.vscode
66
.vscode/*
77
wip
8+
/.benchmarkci
9+
/benchmark/*.json
10+
*.json
11+
*.json.tmp

‎README.md

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,3 +59,13 @@ using Preferences, UUIDs
5959
Preferences.set_preferences!(UUID("764a87c0-6b3e-53db-9096-fe964310641d"),
6060
"PrecompileShooting" => false)
6161
```
62+
63+
## Running Benchmarks Locally
64+
65+
We include a small set of benchmarks in the `benchmarks` folder. These are not extensive and mainly used to track regressions during development. For more extensive benchmarks, see the [SciMLBenchmarks](https://github.com/SciML/SciMLBenchmarks.jl) repository.
66+
67+
To run benchmarks locally install [AirspeedVelocity.jl](https://github.com/MilesCranmer/AirspeedVelocity.jl) and run the following command in the package directory:
68+
69+
```bash
70+
benchpkg BoundaryValueDiffEq --rev="master,<git sha for your commit>" --bench-on="master"
71+
```

‎benchmark/Project.toml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
[deps]
2+
BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
3+
ODEInterface = "54ca160b-1b9f-5127-a996-1867f4bc2a2c"
4+
OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed"

‎benchmark/benchmarks.jl

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
using BenchmarkTools
2+
using BoundaryValueDiffEq, OrdinaryDiffEq
3+
4+
include("simple_pendulum.jl")
5+
6+
function create_benchmark()
7+
suite = BenchmarkGroup()
8+
suite["Simple Pendulum"] = create_simple_pendulum_benchmark()
9+
return suite
10+
end
11+
12+
const SUITE = create_benchmark()

‎benchmark/simple_pendulum.jl

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
module SimplePendulumBenchmark
2+
3+
using BoundaryValueDiffEq, OrdinaryDiffEq
4+
5+
const tspan = (0.0, π / 2)
6+
7+
function simple_pendulum!(du, u, p, t)
8+
g, L, θ, dθ = 9.81, 1.0, u[1], u[2]
9+
du[1] = dθ
10+
du[2] = -(g / L) * sin(θ)
11+
return nothing
12+
end
13+
14+
function bc_pendulum!(residual, u, p, t)
15+
t0, t1 = tspan
16+
residual[1] = u((t0 + t1) / 2)[1] + π / 2
17+
residual[2] = u(t1)[1] - π / 2
18+
return nothing
19+
end
20+
21+
function simple_pendulum(u, p, t)
22+
g, L, θ, dθ = 9.81, 1.0, u[1], u[2]
23+
return [dθ, -(g / L) * sin(θ)]
24+
end
25+
26+
function bc_pendulum(u, p, t)
27+
t0, t1 = tspan
28+
return [u((t0 + t1) / 2)[1] + π / 2, u(t1)[1] - π / 2]
29+
end
30+
31+
const prob_oop = BVProblem{false}(simple_pendulum, bc_pendulum, [π / 2, π / 2], tspan)
32+
const prob_iip = BVProblem{true}(simple_pendulum!, bc_pendulum!, [π / 2, π / 2], tspan)
33+
34+
end
35+
36+
function create_simple_pendulum_benchmark()
37+
suite = BenchmarkGroup()
38+
39+
iip_suite = BenchmarkGroup()
40+
oop_suite = BenchmarkGroup()
41+
42+
suite["IIP"] = iip_suite
43+
suite["OOP"] = oop_suite
44+
45+
if @isdefined(MultipleShooting)
46+
iip_suite["MultipleShooting(100, Tsit5; grid_coarsening = true)"] = @benchmarkable solve($SimplePendulumBenchmark.prob_iip,
47+
$MultipleShooting(100, Tsit5()))
48+
iip_suite["MultipleShooting(100, Tsit5; grid_coarsening = false)"] = @benchmarkable solve($SimplePendulumBenchmark.prob_iip,
49+
$MultipleShooting(100, Tsit5(); grid_coarsening = false))
50+
iip_suite["MultipleShooting(10, Tsit5; grid_coarsening = true)"] = @benchmarkable solve($SimplePendulumBenchmark.prob_iip,
51+
$MultipleShooting(10, Tsit5()))
52+
iip_suite["MultipleShooting(10, Tsit5; grid_coarsening = false)"] = @benchmarkable solve($SimplePendulumBenchmark.prob_iip,
53+
$MultipleShooting(10, Tsit5(); grid_coarsening = false))
54+
end
55+
if @isdefined(Shooting)
56+
iip_suite["Shooting(Tsit5())"] = @benchmarkable solve($SimplePendulumBenchmark.prob_iip,
57+
$Shooting(Tsit5()))
58+
end
59+
60+
if @isdefined(MultipleShooting)
61+
oop_suite["MultipleShooting(100, Tsit5; grid_coarsening = true)"] = @benchmarkable solve($SimplePendulumBenchmark.prob_oop,
62+
$MultipleShooting(100, Tsit5()))
63+
oop_suite["MultipleShooting(100, Tsit5; grid_coarsening = false)"] = @benchmarkable solve($SimplePendulumBenchmark.prob_oop,
64+
$MultipleShooting(100, Tsit5(); grid_coarsening = false))
65+
oop_suite["MultipleShooting(10, Tsit5; grid_coarsening = true)"] = @benchmarkable solve($SimplePendulumBenchmark.prob_oop,
66+
$MultipleShooting(10, Tsit5()))
67+
oop_suite["MultipleShooting(10, Tsit5; grid_coarsening = false)"] = @benchmarkable solve($SimplePendulumBenchmark.prob_oop,
68+
$MultipleShooting(10, Tsit5(); grid_coarsening = false))
69+
end
70+
if @isdefined(Shooting)
71+
oop_suite["Shooting(Tsit5())"] = @benchmarkable solve($SimplePendulumBenchmark.prob_oop,
72+
$Shooting(Tsit5()))
73+
end
74+
75+
return suite
76+
end

0 commit comments

Comments
 (0)