Skip to content

Commit 8129340

Browse files
penelopeysmpenelopeysm
authored
Merge branch 'breaking' into py/params-from-ldf
2 parents 2fad97b + 1d21728 commit 8129340

File tree

9 files changed

+325
-180
lines changed

9 files changed

+325
-180
lines changed

‎.github/workflows/Benchmarking.yml‎

Lines changed: 79 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -4,80 +4,122 @@ on:
44
pull_request:
55

66
jobs:
7-
benchmarks:
7+
benchmark-base:
88
runs-on: ubuntu-latest
9+
outputs:
10+
results: ${{ steps.benchmark.outputs.results }}
11+
sha: ${{ steps.benchmark.outputs.sha }}
12+
steps:
13+
- uses: actions/checkout@v5
14+
with:
15+
ref: ${{ github.base_ref }}
16+
- uses: julia-actions/setup-julia@v2
17+
with:
18+
version: '1.11'
19+
- uses: julia-actions/cache@v2
20+
21+
- name: Run benchmarks
22+
id: benchmark
23+
working-directory: ./benchmarks
24+
run: |
25+
# github output can't handle more than 1 line, hence the tail
26+
julia --project=. -e 'using Pkg; Pkg.instantiate()'
27+
results=$(julia --project=. benchmarks.jl json | tail -n 1 || true)
28+
echo $results
29+
echo "results=$results" >> "$GITHUB_OUTPUT"
30+
echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
931
32+
benchmark-head:
33+
runs-on: ubuntu-latest
34+
outputs:
35+
results: ${{ steps.benchmark.outputs.results }}
36+
sha: ${{ steps.benchmark.outputs.sha }}
1037
steps:
11-
- name: Checkout Repository
12-
uses: actions/checkout@v5
38+
- uses: actions/checkout@v5
1339
with:
1440
ref: ${{ github.event.pull_request.head.sha }}
15-
16-
- name: Set up Julia
17-
uses: julia-actions/setup-julia@v2
41+
- uses: julia-actions/setup-julia@v2
1842
with:
1943
version: '1.11'
20-
2144
- uses: julia-actions/cache@v2
2245

23-
- name: Install Dependencies
24-
run: julia --project=benchmarks/ -e 'using Pkg; Pkg.instantiate()'
46+
- name: Run benchmarks
47+
id: benchmark
48+
working-directory: ./benchmarks
49+
run: |
50+
# github output can't handle more than 1 line, hence the tail
51+
julia --project=. -e 'using Pkg; Pkg.instantiate()'
52+
results=$(julia --project=. benchmarks.jl json | tail -n 1 || true)
53+
echo $results
54+
echo "results=$results" >> "$GITHUB_OUTPUT"
55+
echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
2556
26-
- name: Run Benchmarks
27-
id: run_benchmarks
57+
combine-results:
58+
runs-on: ubuntu-latest
59+
needs: [benchmark-base, benchmark-head]
60+
steps:
61+
- uses: actions/checkout@v5
62+
with:
63+
ref: ${{ github.event.pull_request.head.sha }}
64+
- uses: julia-actions/setup-julia@v2
65+
with:
66+
version: '1.11'
67+
- uses: julia-actions/cache@v2
68+
69+
- name: Combine benchmark results
70+
working-directory: ./benchmarks
2871
run: |
29-
# Capture version info into a variable, print it, and set it as an env var for later steps
3072
version_info=$(julia -e 'using InteractiveUtils; versioninfo()')
3173
echo "$version_info"
3274
echo "VERSION_INFO<<EOF" >> $GITHUB_ENV
3375
echo "$version_info" >> $GITHUB_ENV
3476
echo "EOF" >> $GITHUB_ENV
3577
36-
# Capture benchmark output into a variable. The sed and tail calls cut out anything but the
37-
# final block of results.
38-
echo "Running Benchmarks..."
39-
benchmark_output=$(\
40-
julia --project=benchmarks benchmarks/benchmarks.jl \
41-
| sed -n '/Final results:/,$p' \
42-
| tail -n +2\
43-
)
78+
# save outputs of previous jobs to json file
79+
echo "Base results"
80+
echo "--------------------------------------------------------"
81+
echo '${{needs.benchmark-base.outputs.results}}'
82+
echo '${{needs.benchmark-base.outputs.results}}' > base.json
83+
echo "Head results"
84+
echo "--------------------------------------------------------"
85+
echo '${{needs.benchmark-head.outputs.results}}'
86+
echo '${{needs.benchmark-head.outputs.results}}' > head.json
4487
45-
# Print benchmark results directly to the workflow log
46-
echo "Benchmark Results:"
47-
echo "$benchmark_output"
88+
# combine them and save the output as an env var for later steps
89+
julia --project=. -e 'using Pkg; Pkg.instantiate()'
90+
results=$(julia --project=. benchmarks.jl combine head.json base.json)
91+
echo "Combined results"
92+
echo "--------------------------------------------------------"
93+
echo "$results"
4894
49-
# Set the benchmark output as an env var for later steps
5095
echo "BENCHMARK_OUTPUT<<EOF" >> $GITHUB_ENV
51-
echo "$benchmark_output" >> $GITHUB_ENV
96+
echo "$results" >> $GITHUB_ENV
5297
echo "EOF" >> $GITHUB_ENV
5398
54-
# Get the current commit SHA of DynamicPPL
55-
DPPL_COMMIT_SHA=$(git rev-parse HEAD)
56-
echo "DPPL_COMMIT_SHA=$DPPL_COMMIT_SHA" >> $GITHUB_ENV
57-
58-
COMMIT_URL="https://github.com/${{ github.repository }}/commit/$DPPL_COMMIT_SHA"
59-
echo "DPPL_COMMIT_URL=$COMMIT_URL" >> $GITHUB_ENV
60-
61-
- name: Find Existing Comment
99+
- name: Find existing benchmark comment
62100
uses: peter-evans/find-comment@v4
63101
id: find_comment
64102
with:
65103
issue-number: ${{ github.event.pull_request.number }}
66104
comment-author: github-actions[bot]
67105

68-
- name: Post Benchmark Results as PR Comment
106+
- name: Create or update benchmark comment
69107
uses: peter-evans/create-or-update-comment@v5
70108
with:
71109
issue-number: ${{ github.event.pull_request.number }}
72110
body: |
73-
## Benchmark Report for Commit ${{ env.DPPL_COMMIT_SHA }}
111+
## Benchmark Report
112+
113+
- this PR's head: `${{ needs.benchmark-head.outputs.sha }}`
114+
- base branch: `${{ needs.benchmark-base.outputs.sha }}`
115+
74116
### Computer Information
75117
```
76118
${{ env.VERSION_INFO }}
77119
```
78120
### Benchmark Results
79-
```
121+
80122
${{ env.BENCHMARK_OUTPUT }}
81-
```
123+
82124
comment-id: ${{ steps.find_comment.outputs.comment-id }}
83125
edit-mode: replace

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,5 @@
44
.DS_Store
55
Manifest.toml
66
**.~undo-tree~
7+
8+
benchmarks/*.json

‎benchmarks/Project.toml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,12 @@ version = "0.1.0"
44

55
[deps]
66
ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
7-
BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
7+
Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de"
88
Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
99
DynamicPPL = "366bfd00-2699-11ea-058f-f148b4cae6d8"
1010
Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9"
1111
ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
12+
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
1213
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
1314
LogDensityProblems = "6fdf6af0-433a-55f7-b3ed-c6c6e0b8df7c"
1415
Mooncake = "da2b9cff-9c12-43a0-ae48-6db2b0edb7d6"
@@ -21,11 +22,12 @@ DynamicPPL = {path = "../"}
2122

2223
[compat]
2324
ADTypes = "1.14.0"
24-
BenchmarkTools = "1.6.0"
25+
Chairmarks = "1.3.1"
2526
Distributions = "0.25.117"
2627
DynamicPPL = "0.39"
2728
Enzyme = "0.13"
28-
ForwardDiff = "0.10.38, 1"
29+
ForwardDiff = "1"
30+
JSON = "1.3.0"
2931
LogDensityProblems = "2.1.2"
3032
Mooncake = "0.4"
3133
PrettyTables = "3"

‎benchmarks/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
To run the benchmarks, run this from the root directory of the repository:
1+
To run the benchmarks locally, run this from the root directory of the repository:
22

33
```sh
44
julia --project=benchmarks benchmarks/benchmarks.jl

0 commit comments

Comments
 (0)