Skip to content

Commit 3c0e5b7

Browse files
findmywayfindmyway
authored
Support BenchmarkTools.jl in Julia (#89)
1 parent 3f97db6 commit 3c0e5b7

File tree

14 files changed

+393
-0
lines changed

14 files changed

+393
-0
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,45 @@ jobs:
203203
fail-on-alert: true
204204
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark'
205205

206+
julia-benchmark:
207+
name: Run Julia benchmark example
208+
runs-on: ubuntu-latest
209+
steps:
210+
- uses: actions/checkout@v2
211+
- uses: actions/setup-node@v1
212+
- uses: actions/cache@v1
213+
with:
214+
path: ~/.npm
215+
key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
216+
- run: npm ci
217+
- run: npm run build
218+
- name: Save previous data.js
219+
run: |
220+
git fetch origin gh-pages
221+
git checkout gh-pages
222+
cp ./dev/bench/data.js before_data.js
223+
git checkout -
224+
- uses: julia-actions/setup-julia@v1
225+
with:
226+
version: '1'
227+
arch: x64
228+
- name: Run benchmark
229+
run: |
230+
cd examples/julia
231+
julia --project --color=yes -e '
232+
using Pkg;
233+
Pkg.instantiate();
234+
include("fib.jl")'
235+
- name: Store benchmark result
236+
uses: ./
237+
with:
238+
name: Julia benchmark
239+
tool: "julia"
240+
output-file-path: examples/julia/output.json
241+
skip-fetch-gh-pages: true
242+
fail-on-alert: true
243+
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark'
244+
206245
only-alert-with-cache:
207246
name: Run alert check with actions/cache
208247
runs-on: ubuntu-latest

‎.github/workflows/julia.yml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
name: Julia Example with BenchmarkTools.jl
2+
on:
3+
push:
4+
branches:
5+
- master
6+
7+
permissions:
8+
contents: write
9+
deployments: write
10+
11+
jobs:
12+
benchmark:
13+
name: Run julia benchmark example
14+
runs-on: ${{ matrix.os }}
15+
strategy:
16+
fail-fast: false
17+
matrix:
18+
version:
19+
- '1'
20+
os:
21+
- ubuntu-latest
22+
arch:
23+
- x64
24+
steps:
25+
- uses: actions/checkout@v2
26+
- uses: julia-actions/setup-julia@v1
27+
with:
28+
version: ${{ matrix.version }}
29+
arch: ${{ matrix.arch }}
30+
- uses: actions/cache@v1
31+
env:
32+
cache-name: cache-artifacts
33+
with:
34+
path: ~/.julia/artifacts
35+
key: runner.os−test−env.cache−name−{{ hashFiles('**/Project.toml') }}
36+
restore-keys: |
37+
runner.os−test−
38+
${{ env.cache-name }}-
39+
${{ runner.os }}-test-
40+
${{ runner.os }}-
41+
- name: Run benchmark
42+
run: |
43+
cd examples/julia
44+
julia --project --color=yes -e '
45+
using Pkg;
46+
Pkg.instantiate();
47+
include("fib.jl")'
48+
- name: Store benchmark result
49+
uses: benchmark-action/github-action-benchmark@v1
50+
with:
51+
name: Julia benchmark result
52+
tool: 'julia'
53+
output-file-path: examples/julia/output.json
54+
# Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096
55+
github-token: ${{ secrets.GITHUB_TOKEN }}
56+
auto-push: true
57+
# Show alert with commit comment on detecting possible performance regression
58+
alert-threshold: '200%'
59+
comment-on-alert: true
60+
fail-on-alert: true
61+
alert-comment-cc-users: '@findmyway'

‎README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ This action currently supports the following tools:
2222
- [pytest-benchmark][] for Python projects with [pytest][]
2323
- [Google Benchmark Framework][google-benchmark] for C++ projects
2424
- [Catch2][catch2] for C++ projects
25+
- [BenchmarkTools.jl][] for Julia packages
2526
- Custom benchmarks where either 'biggerIsBetter' or 'smallerIsBetter'
2627

2728
Multiple languages in the same repository are supported for polyglot projects.
@@ -43,6 +44,7 @@ definitions are in [.github/workflows/](./.github/workflows) directory. Live wor
4344
| Python | [![pytest-benchmark Example Workflow][pytest-benchmark-badge]][pytest-workflow-example] | [examples/pytest](./examples/pytest) |
4445
| C++ | [![C++ Example Workflow][cpp-badge]][cpp-workflow-example] | [examples/cpp](./examples/cpp) |
4546
| C++ (Catch2) | [![C++ Catch2 Example Workflow][catch2-badge]][catch2-workflow-example] | [examples/catch2](./examples/catch2) |
47+
| Julia | [![Julia Example][julia-badge]][catch2-workflow-example] | [examples/julia](./examples/julia) |
4648

4749
All benchmark charts from above workflows are gathered in GitHub pages:
4850

@@ -580,6 +582,7 @@ Every release will appear on your GitHub notifications page.
580582
[pytest-benchmark-badge]: https://github.com/benchmark-action/github-action-benchmark/workflows/Python%20Example%20with%20pytest/badge.svg
581583
[cpp-badge]: https://github.com/benchmark-action/github-action-benchmark/workflows/C%2B%2B%20Example/badge.svg
582584
[catch2-badge]: https://github.com/benchmark-action/github-action-benchmark/workflows/Catch2%20C%2B%2B%20Example/badge.svg
585+
[julia-badge]: https://github.com/benchmark-action/github-action-benchmark/workflows/Julia Example%20with%20BenchmarkTools.jl/badge.svg
583586
[github-action]: https://github.com/features/actions
584587
[cargo-bench]: https://doc.rust-lang.org/cargo/commands/cargo-bench.html
585588
[benchmarkjs]: https://benchmarkjs.com/
@@ -602,3 +605,4 @@ Every release will appear on your GitHub notifications page.
602605
[catch2]: https://github.com/catchorg/Catch2
603606
[lighthouse-ci-action]: https://github.com/treosh/lighthouse-ci-action
604607
[lighthouse-ci]: https://github.com/GoogleChrome/lighthouse-ci
608+
[BenchmarkTools.jl]: https://github.com/JuliaCI/BaseBenchmarks.jl

‎examples/julia/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
/output.json

‎examples/julia/Manifest.toml‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
# This file is machine-generated - editing it directly is not advised
2+
3+
julia_version = "1.7.0-rc3"
4+
manifest_format = "2.0"
5+
6+
[[deps.Artifacts]]
7+
uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
8+
9+
[[deps.BenchmarkTools]]
10+
deps = ["JSON", "Logging", "Printf", "Profile", "Statistics", "UUIDs"]
11+
git-tree-sha1 = "61adeb0823084487000600ef8b1c00cc2474cd47"
12+
uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
13+
version = "1.2.0"
14+
15+
[[deps.CompilerSupportLibraries_jll]]
16+
deps = ["Artifacts", "Libdl"]
17+
uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
18+
19+
[[deps.Dates]]
20+
deps = ["Printf"]
21+
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
22+
23+
[[deps.JSON]]
24+
deps = ["Dates", "Mmap", "Parsers", "Unicode"]
25+
git-tree-sha1 = "8076680b162ada2a031f707ac7b4953e30667a37"
26+
uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
27+
version = "0.21.2"
28+
29+
[[deps.Libdl]]
30+
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
31+
32+
[[deps.LinearAlgebra]]
33+
deps = ["Libdl", "libblastrampoline_jll"]
34+
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
35+
36+
[[deps.Logging]]
37+
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
38+
39+
[[deps.Mmap]]
40+
uuid = "a63ad114-7e13-5084-954f-fe012c677804"
41+
42+
[[deps.OpenBLAS_jll]]
43+
deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
44+
uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
45+
46+
[[deps.Parsers]]
47+
deps = ["Dates"]
48+
git-tree-sha1 = "ae4bbcadb2906ccc085cf52ac286dc1377dceccc"
49+
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
50+
version = "2.1.2"
51+
52+
[[deps.Printf]]
53+
deps = ["Unicode"]
54+
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
55+
56+
[[deps.Profile]]
57+
deps = ["Printf"]
58+
uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"
59+
60+
[[deps.Random]]
61+
deps = ["SHA", "Serialization"]
62+
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
63+
64+
[[deps.SHA]]
65+
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
66+
67+
[[deps.Serialization]]
68+
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
69+
70+
[[deps.SparseArrays]]
71+
deps = ["LinearAlgebra", "Random"]
72+
uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
73+
74+
[[deps.Statistics]]
75+
deps = ["LinearAlgebra", "SparseArrays"]
76+
uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
77+
78+
[[deps.UUIDs]]
79+
deps = ["Random", "SHA"]
80+
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
81+
82+
[[deps.Unicode]]
83+
uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
84+
85+
[[deps.libblastrampoline_jll]]
86+
deps = ["Artifacts", "Libdl", "OpenBLAS_jll"]
87+
uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"

‎examples/julia/Project.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[deps]
2+
BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"

‎examples/julia/README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Julia example with `BenchmarkTools.jl`
2+
3+
Please read the [docs](https://juliaci.github.io/BenchmarkTools.jl/stable/manual/) of `BenchmarkTools.jl` first. Expecially the [BenchmarkGroup](https://juliaci.github.io/BenchmarkTools.jl/stable/manual/#The-BenchmarkGroup-type) section. Generally speaking, we only need the `json` file exported by `BenchmarkTools.save`. You can checkout the [`fib.jl`](./fib.jl) file for how to do it. A [workflow](../../.github/workflows/julia.yml) for this example is also provided to help you integrate it in your project.
4+
5+
**Note:** Currently we only support test suite after applying an estimation (`minimumm`,`median`, `mean`, `maximum`, `std`).

‎examples/julia/fib.jl‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
using BenchmarkTools
2+
3+
fib(n) = n <= 1 ? 1 : fib(n - 2) + fib(n - 1)
4+
5+
suite = BenchmarkGroup()
6+
7+
suite["fib"] = BenchmarkGroup(["tag1", "tag2"])
8+
9+
suite["fib"][10] = @benchmarkable fib(10)
10+
suite["fib"][20] = @benchmarkable fib(20)
11+
12+
tune!(suite)
13+
results = run(suite, verbose = true)
14+
15+
BenchmarkTools.save("output.json", median(results))

‎src/config.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ export type ToolType =
1010
| 'pytest'
1111
| 'googlecpp'
1212
| 'catch2'
13+
| 'julia'
1314
| 'customBiggerIsBetter'
1415
| 'customSmallerIsBetter';
1516
export interface Config {
@@ -39,6 +40,7 @@ export const VALID_TOOLS: ToolType[] = [
3940
'pytest',
4041
'googlecpp',
4142
'catch2',
43+
'julia',
4244
'customBiggerIsBetter',
4345
'customSmallerIsBetter',
4446
];

‎src/default_index_html.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ export const DEFAULT_INDEX_HTML = String.raw`<!DOCTYPE html>
117117
pytest: '#3572a5',
118118
googlecpp: '#f34b7d',
119119
catch2: '#f34b7d',
120+
julia: '#a270ba',
120121
customBiggerIsBetter: '#38ff38',
121122
customSmallerIsBetter: '#ff3838',
122123
_: '#333333'

0 commit comments

Comments
 (0)