Skip to content

Commit 60d9aa0

Browse files
cameelcameel
committed
Benchmark report collector job + summary
1 parent c6094bb commit 60d9aa0

File tree

3 files changed

+148
-0
lines changed

3 files changed

+148
-0
lines changed

‎.circleci/config.yml

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1216,6 +1216,23 @@ jobs:
12161216
- reports/externalTests/
12171217
- gitter_notify_failure_unless_pr
12181218

1219+
c_ext_benchmarks:
1220+
<<: *base_node_small
1221+
steps:
1222+
- checkout
1223+
- attach_workspace:
1224+
at: .
1225+
- run:
1226+
name: Combine benchmark reports
1227+
command: cat reports/externalTests/benchmark-*.json | scripts/externalTests/merge_benchmarks.sh > reports/externalTests/all-benchmarks.json
1228+
- run:
1229+
name: Summarize reports
1230+
command: cat reports/externalTests/all-benchmarks.json | scripts/externalTests/summarize_benchmarks.sh > reports/externalTests/summarized-benchmarks.json
1231+
- store_artifacts:
1232+
path: reports/externalTests/all-benchmarks.json
1233+
- store_artifacts:
1234+
path: reports/externalTests/summarized-benchmarks.json
1235+
12191236
b_win: &b_win
12201237
<<: *base_win_powershell_large
12211238
steps:
@@ -1466,6 +1483,24 @@ workflows:
14661483
- t_ems_ext: *job_native_test_ext_prb_math
14671484
- t_ems_ext: *job_native_test_ext_elementfi
14681485

1486+
- c_ext_benchmarks:
1487+
<<: *workflow_trigger_on_tags
1488+
requires:
1489+
- t_ems_compile_ext_colony
1490+
- t_native_compile_ext_gnosis
1491+
- t_native_test_ext_gnosis_v2
1492+
- t_native_test_ext_zeppelin
1493+
- t_native_test_ext_ens
1494+
- t_native_test_ext_trident
1495+
- t_native_test_ext_euler
1496+
- t_native_test_ext_yield_liquidator
1497+
- t_native_test_ext_bleeps
1498+
- t_native_test_ext_pool_together
1499+
- t_native_test_ext_perpetual_pools
1500+
- t_native_test_ext_uniswap
1501+
- t_native_test_ext_prb_math
1502+
- t_native_test_ext_elementfi
1503+
14691504
# Windows build and tests
14701505
- b_win: *workflow_trigger_on_tags
14711506
- b_win_release: *workflow_trigger_on_tags

‎scripts/externalTests/merge_benchmarks.sh

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!/usr/bin/env bash
2+
3+
# ------------------------------------------------------------------------------
4+
# Reads multiple individual benchmark reports produced by scripts from
5+
# test/externalTests/ from standard input and creates a combined report.
6+
#
7+
# Usage:
8+
# <script name>.sh < <CONCATENATED_REPORTS>
9+
#
10+
# CONCATENATED_REPORTS: JSON report files concatenated into a single stream (e.g. using cat).
11+
#
12+
# Example:
13+
# cat reports/externalTests/benchmark-*.json | <script name>.sh
14+
# ------------------------------------------------------------------------------
15+
# This file is part of solidity.
16+
#
17+
# solidity is free software: you can redistribute it and/or modify
18+
# it under the terms of the GNU General Public License as published by
19+
# the Free Software Foundation, either version 3 of the License, or
20+
# (at your option) any later version.
21+
#
22+
# solidity is distributed in the hope that it will be useful,
23+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
24+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25+
# GNU General Public License for more details.
26+
#
27+
# You should have received a copy of the GNU General Public License
28+
# along with solidity. If not, see <http://www.gnu.org/licenses/>
29+
#
30+
# (c) 2021 solidity contributors.
31+
#------------------------------------------------------------------------------
32+
33+
set -euo pipefail
34+
35+
# We expect a series of dicts of the form {"<project>": {"<preset>": {...}}}.
36+
# Unfortunately jq's built-in `add` filter can't handle nested dicts and
37+
# would just overwrite values sharing a project name instead of merging them.
38+
39+
# This is done by first grouping the dicts into an array of the form
40+
# [
41+
# [{"key": "<project1>", "value": {"<preset1>": {...}}}, {"key": "<project1>", "value": {"<preset2>": {...}}, ...],
42+
# [{"key": "<project2>", "value": {"<preset1>": {...}}}, {"key": "<project2>", "value": {"<preset2>": {...}}, ...],
43+
# ...
44+
# ]
45+
# and then using reduce() on each group sharing the same project name to convert it into a
46+
# dict having preset names as keys.
47+
jq --slurp --indent 4 --sort-keys '
48+
map(to_entries[]) |
49+
group_by(.key) |
50+
map({
51+
(.[0].key): (
52+
reduce (.[].value | to_entries[]) as {$key, $value} (
53+
{}; . + {
54+
($key): $value
55+
}
56+
)
57+
)
58+
}) |
59+
add
60+
'

‎scripts/externalTests/summarize_benchmarks.sh

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#!/usr/bin/env bash
2+
3+
# ------------------------------------------------------------------------------
4+
# Reads a combined benchmark report from standard input and outputs an abbreviated
5+
# report containing only totals. Can handle individual reports coming directly
6+
# from scripts in test/externalTests/ as well as combined report from merge_benchmarks.sh.
7+
#
8+
# Usage:
9+
# <script name>.sh < <CONCATENATED_REPORTS>
10+
#
11+
# CONCATENATED_REPORTS: JSON report files concatenated into a single stream (e.g. using cat).
12+
#
13+
# Example:
14+
# cat reports/externalTests/benchmark-*.json | <script name>.sh
15+
# ------------------------------------------------------------------------------
16+
# This file is part of solidity.
17+
#
18+
# solidity is free software: you can redistribute it and/or modify
19+
# it under the terms of the GNU General Public License as published by
20+
# the Free Software Foundation, either version 3 of the License, or
21+
# (at your option) any later version.
22+
#
23+
# solidity is distributed in the hope that it will be useful,
24+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
25+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26+
# GNU General Public License for more details.
27+
#
28+
# You should have received a copy of the GNU General Public License
29+
# along with solidity. If not, see <http://www.gnu.org/licenses/>
30+
#
31+
# (c) 2021 solidity contributors.
32+
#------------------------------------------------------------------------------
33+
34+
set -euo pipefail
35+
36+
REPO_ROOT=$(realpath "$(dirname "$0")/../..")
37+
38+
# Iterates over presets in a dict of the form {"<project>": {"<preset>": {...}}} and for each
39+
# one preserves only the few keys with totals that we want to see in the summary.
40+
exec "${REPO_ROOT}/scripts/externalTests/merge_benchmarks.sh" | jq --indent 4 --sort-keys '
41+
with_entries({
42+
key: .key,
43+
value: .value | with_entries({
44+
key: .key,
45+
value: {
46+
bytecode_size: .value.total_bytecode_size,
47+
method_gas: .value.gas.total_method_gas,
48+
deployment_gas: .value.gas.total_deployment_gas,
49+
version: .value.project.version
50+
}
51+
})
52+
})
53+
'

0 commit comments

Comments
 (0)