Skip to content

Commit 7fa5a9e

Browse files
baronfelspl
baronfel
authored and
spl
committed
add support for benchmark.net json result files
Source: benchmark-action/github-action-benchmark#34
1 parent c076a65 commit 7fa5a9e

File tree

15 files changed

+1831
-11
lines changed

15 files changed

+1831
-11
lines changed

‎.github/workflows/benchmarkdotnet.yml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
name: Benchmark.Net Example
2+
on:
3+
push:
4+
branches:
5+
- master
6+
7+
jobs:
8+
benchmark:
9+
name: Run Benchmark.Net benchmark example
10+
runs-on: ubuntu-latest
11+
steps:
12+
- uses: actions/checkout@v2
13+
- uses: actions/setup-dotnet@v1
14+
with:
15+
dotnet-version: '3.1.201' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json
16+
- name: Run benchmark
17+
run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*'
18+
- name: Store benchmark result
19+
uses: rhysd/github-action-benchmark@v1
20+
with:
21+
name: Benchmark.Net Benchmark
22+
tool: 'benchmarkdotnet'
23+
output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json
24+
# Use personal access token instead of GITHUB_TOKEN due to https://github.community/t5/GitHub-Actions/Github-action-not-triggering-gh-pages-upon-push/td-p/26869/highlight/false
25+
github-token: ${{ secrets.PERSONAL_GITHUB_TOKEN }}
26+
auto-push: true
27+
# Show alert with commit comment on detecting possible performance regression
28+
alert-threshold: '200%'
29+
comment-on-alert: true
30+
fail-on-alert: true
31+
alert-comment-cc-users: '@rhysd'

‎.github/workflows/ci.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,38 @@ jobs:
200200
skip-fetch-gh-pages: true
201201
fail-on-alert: true
202202
- run: test -f before_data.js && node ./scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' || true
203+
benchmarkdotnet-framework:
204+
name: Run Benchmark.Net example
205+
runs-on: ubuntu-latest
206+
steps:
207+
- uses: actions/checkout@v2
208+
- uses: actions/setup-dotnet@v1
209+
with:
210+
dotnet-version: '3.1.201' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json
211+
- uses: actions/cache@v1
212+
with:
213+
path: ~/.npm
214+
key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
215+
- run: npm ci
216+
- run: npm run build
217+
- name: Save previous data.js
218+
run: |
219+
git fetch origin gh-pages
220+
git checkout gh-pages
221+
# If this is the first run, data.js will not there.
222+
cp ./dev/bench/data.js before_data.js || true
223+
git checkout -
224+
- name: Run benchmark
225+
run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*'
226+
- name: Store benchmark result
227+
uses: ./
228+
with:
229+
name: Benchmark.Net Benchmark
230+
tool: "benchmarkdotnet"
231+
output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json
232+
skip-fetch-gh-pages: true
233+
fail-on-alert: true
234+
- run: test -f before_data.js && node ./scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' || true
203235

204236
only-alert-with-cache:
205237
name: Run alert check with actions/cache

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55
/examples/rust/target
66
/examples/criterion-rs/Cargo.lock
77
/examples/criterion-rs/target
8+
/examples/benchmarkdotnet/bin
9+
/examples/benchmarkdotnet/obj
10+
/examples/benchmarkdotnet/BenchmarkDotNet.Artifacts
811
/test/*.js
912
/test/*.js.map
1013
/scripts/*.js

‎README.md‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ This action currently supports the following tools:
3131
- [pytest-benchmark][] for Python projects with [pytest][]
3232
- [Google Benchmark Framework][google-benchmark] for C++ projects
3333
- [Catch2][catch2] for C++ projects
34+
- [Benchmark.Net][benchmarkdotnet] for .Net projects
3435

3536
Multiple languages in the same repository are supported for polyglot projects.
3637

@@ -43,14 +44,15 @@ Multiple languages in the same repository are supported for polyglot projects.
4344
Example projects for each language are in [examples/](./examples) directory. Live example workflow
4445
definitions are in [.github/workflows/](./.github/workflows) directory. Live workflows are:
4546

46-
| Language | Workflow | Example Project |
47-
|--------------|-----------------------------------------------------------------------------------------|------------------------------------------------|
48-
| Rust | [![Rust Example Workflow][rust-badge]][rust-workflow-example] | [examples/rust](./examples/rust) |
49-
| Go | [![Go Example Workflow][go-badge]][go-workflow-example] | [examples/go](./examples/go) |
50-
| JavaScript | [![JavaScript Example Workflow][benchmarkjs-badge]][benchmarkjs-workflow-example] | [examples/benchmarkjs](./examples/benchmarkjs) |
51-
| Python | [![pytest-benchmark Example Workflow][pytest-benchmark-badge]][pytest-workflow-example] | [examples/pytest](./examples/pytest) |
52-
| C++ | [![C++ Example Workflow][cpp-badge]][cpp-workflow-example] | [examples/cpp](./examples/cpp) |
53-
| C++ (Catch2) | [![C++ Catch2 Example Workflow][catch2-badge]][catch2-workflow-example] | [examples/catch2](./examples/catch2) |
47+
| Language | Workflow | Example Project |
48+
|--------------|-------------------------------------------------------------------------------------------------|--------------------------------------------------------|
49+
| Rust | [![Rust Example Workflow][rust-badge]][rust-workflow-example] | [examples/rust](./examples/rust) |
50+
| Go | [![Go Example Workflow][go-badge]][go-workflow-example] | [examples/go](./examples/go) |
51+
| JavaScript | [![JavaScript Example Workflow][benchmarkjs-badge]][benchmarkjs-workflow-example] | [examples/benchmarkjs](./examples/benchmarkjs) |
52+
| Python | [![pytest-benchmark Example Workflow][pytest-benchmark-badge]][pytest-workflow-example] | [examples/pytest](./examples/pytest) |
53+
| C++ | [![C++ Example Workflow][cpp-badge]][cpp-workflow-example] | [examples/cpp](./examples/cpp) |
54+
| C++ (Catch2) | [![C++ Catch2 Example Workflow][catch2-badge]][catch2-workflow-example] | [examples/catch2](./examples/catch2) |
55+
| .Net | [![C# Benchmark.Net Example Workflow][benchmarkdotnet-badge]][benchmarkdotnet-workflow-example] | [examples/benchmarkdotnet](./examples/benchmarkdotnet) |
5456

5557
All benchmark charts from above workflows are gathered in GitHub pages:
5658

@@ -311,6 +313,7 @@ and store it to file. Then specify the file path to `output-file-path` input.
311313
- [Benchmark.js for JavaScript/TypeScript projects](./examples/benchmarkjs/README.md)
312314
- [pytest-benchmark for Python projects with pytest](./examples/pytest/README.md)
313315
- [Google Benchmark Framework for C++ projects](./examples/cpp/README.md)
316+
- [Benchmark.Net for .Net projects](./examples/benchmarkdotnet/README.md)
314317

315318
These examples are run in workflows of this repository as described in the 'Examples' section above.
316319

@@ -332,7 +335,7 @@ Name of the benchmark. This value must be identical across all benchmarks in you
332335
- Default: N/A
333336

334337
Tool for running benchmark. The value must be one of `"cargo"`, `"go"`, `"benchmarkjs"`, `"pytest"`,
335-
`"googlecpp"`, `"catch2"`.
338+
`"googlecpp"`, `"catch2"`, `"benchmarkdotnet"`.
336339

337340
#### `output-file-path` (Required)
338341

@@ -599,3 +602,6 @@ will appear on your GitHub notifications page.
599602
[catch2]: https://github.com/catchorg/Catch2
600603
[lighthouse-ci-action]: https://github.com/treosh/lighthouse-ci-action
601604
[lighthouse-ci]: https://github.com/GoogleChrome/lighthouse-ci
605+
[benchmarkdotnet]: [https://benchmarkdotnet.org]
606+
[benchmarkdotnet-badge]: [https://github.com/rhysd/github-action-benchmark/workflows/Benchmark.Net%20Example/badge.svg]
607+
[benchmarkdotnet-workflow-example]: [https://github.com/rhysd/github-action-benchmark/actions?query=workflow%3A%22Benchmark.Net+Example%22]

‎examples/benchmarkdotnet/Benchmarks.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
using BenchmarkDotNet.Attributes;
2+
3+
namespace Sample
4+
{
5+
public class Benchmarks
6+
{
7+
public static int Fib(int n) {
8+
switch (n)
9+
{
10+
case 0: return 0;
11+
case 1: return 1;
12+
default:
13+
return Fib(n-2) + Fib(2-1);
14+
}
15+
}
16+
17+
[Benchmark]
18+
public void Fib10() => Fib(10);
19+
20+
[Benchmark]
21+
public void Fib20() => Fib(20);
22+
}
23+
}

‎examples/benchmarkdotnet/Program.cs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
using BenchmarkDotNet.Running;
2+
3+
namespace Sample
4+
{
5+
public class Program
6+
{
7+
public static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
8+
}
9+
}

‎examples/benchmarkdotnet/README.md‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
C# example for benchmarking with `Benchmark.Net`
2+
================================================
3+
4+
- [Workflow for this example](../../.github/workflows/benchmarkdotnet.yml)
5+
- [Action log of this example](https://github.com/rhysd/github-action-benchmark/actions?query=workflow%3A%22Benchmark.Net+example%22)
6+
- [Benchmark results on GitHub pages](https://rhysd.github.io/github-action-benchmark/dev/bench/)
7+
8+
This directory shows how to use [`github-action-benchmark`](https://github.com/rhysd/github-action-benchmark)
9+
with [`Benchmark.Net`](https://benchmarkdotnet.org/).
10+
11+
## Run benchmarks
12+
13+
Official documentation for usage of `Benchmark.Net`:
14+
15+
https://benchmarkdotnet.org/articles/overview.html
16+
17+
You should add the `Benchmark.Net` package to your test project and configure your tests according to the [Getting Started](https://benchmarkdotnet.org/articles/guides/getting-started.html) docs. A simple test file might look like
18+
19+
20+
```csharp
21+
using System;
22+
using System.Security.Cryptography;
23+
using BenchmarkDotNet.Attributes;
24+
using BenchmarkDotNet.Running;
25+
26+
namespace MyBenchmarks
27+
{
28+
[JsonExporterAttribute.Full]
29+
[JsonExporterAttribute.FullCompressed]
30+
public class Md5VsSha256
31+
{
32+
private const int N = 10000;
33+
private readonly byte[] data;
34+
35+
private readonly SHA256 sha256 = SHA256.Create();
36+
private readonly MD5 md5 = MD5.Create();
37+
38+
public Md5VsSha256()
39+
{
40+
data = new byte[N];
41+
new Random(42).NextBytes(data);
42+
}
43+
44+
[Benchmark]
45+
public byte[] Sha256() => sha256.ComputeHash(data);
46+
47+
[Benchmark]
48+
public byte[] Md5() => md5.ComputeHash(data);
49+
}
50+
51+
public class Program
52+
{
53+
public static void Main(string[] args)
54+
{
55+
var summary = BenchmarkRunner.Run<Md5VsSha256>();
56+
}
57+
}
58+
}
59+
```
60+
61+
You can then run the tests using `dotnet run`. It's _very_ important that you ensure the JSON exporter is configured. You can do this by adding at least one of the exporter attributes in the example above, or by using the `BenchmarkSwitcher` type to run your tests, passing in your `args`, and using `--exporters json` from the command line.
62+
63+
## Process benchmark results
64+
65+
Store the benchmark results with step using the action. Please set `benchmarkdotnet` to `tool` input.
66+
67+
By default, Benchmark.Net will output results files to the current directory in a structure like:
68+
69+
```
70+
BenchmarkDotNet.Artifacts
71+
├── Sample.Benchmarks-20200529-153703.log
72+
├── Sample.Benchmarks-20200529-153729.log
73+
└── results
74+
├── Sample.Benchmarks-report-full-compressed.json
75+
├── Sample.Benchmarks-report-github.md
76+
├── Sample.Benchmarks-report.csv
77+
└── Sample.Benchmarks-report.html
78+
```
79+
80+
You want to get the path of the `-report-full-compressed.json` report for use with this action. Once you have both pieces of data, use the action like so, replacing the output file path with your own path.
81+
82+
```yaml
83+
- name: Store benchmark result
84+
uses: rhysd/github-action-benchmark@v1
85+
with:
86+
tool: 'benchmarkdotnet'
87+
output-file-path: BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json
88+
```
89+
90+
Please read ['How to use' section](https://github.com/rhysd/github-action-benchmark#how-to-use) for common usage.

‎examples/benchmarkdotnet/Sample.csproj‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<PropertyGroup>
3+
<TargetFramework>netcoreapp3.0</TargetFramework>
4+
<OutputType>Exe</OutputType>
5+
</PropertyGroup>
6+
<PropertyGroup>
7+
<PlatformTarget>AnyCPU</PlatformTarget>
8+
<DebugType>portable</DebugType>
9+
<DebugSymbols>true</DebugSymbols>
10+
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
11+
<Optimize>true</Optimize>
12+
<Configuration>Release</Configuration>
13+
</PropertyGroup>
14+
<ItemGroup>
15+
<PackageReference Include="BenchmarkDotNet" Version="0.12.1" />
16+
</ItemGroup>
17+
</Project>

‎examples/benchmarkdotnet/global.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"sdk": {
3+
"version": "3.1.201"
4+
}
5+
}

‎src/config.ts‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,15 @@ import { promises as fs } from 'fs';
33
import * as os from 'os';
44
import * as path from 'path';
55

6-
export type ToolType = 'generic' | 'cargo' | 'go' | 'benchmarkjs' | 'pytest' | 'googlecpp' | 'catch2';
6+
export type ToolType =
7+
| 'generic'
8+
| 'cargo'
9+
| 'go'
10+
| 'benchmarkjs'
11+
| 'pytest'
12+
| 'googlecpp'
13+
| 'catch2'
14+
| 'benchmarkdotnet';
715
export interface Config {
816
name: string;
917
tool: ToolType;
@@ -24,7 +32,16 @@ export interface Config {
2432
maxItemsInChart: number | null;
2533
}
2634

27-
export const VALID_TOOLS: ToolType[] = ['generic', 'cargo', 'go', 'benchmarkjs', 'pytest', 'googlecpp', 'catch2'];
35+
export const VALID_TOOLS: ToolType[] = [
36+
'generic',
37+
'cargo',
38+
'go',
39+
'benchmarkjs',
40+
'pytest',
41+
'googlecpp',
42+
'catch2',
43+
'benchmarkdotnet',
44+
];
2845
const RE_UINT = /^\d+$/;
2946

3047
function validateToolType(tool: string): asserts tool is ToolType {

0 commit comments

Comments
 (0)