Skip to content

Commit c321210

Browse files
phadejphadej
committed
Add bench.sh and aeson-benchmark-suite
- bench.sh is the runner and comparator for benchmarks we have. - The idea is to migrate all benchmarks we have into `aeson-benchmark-suite` - I added `criterion-compare-txt` to output textual tables, these are easier to copy into pull request etc. - Added GitHub issues example, we still need more.
1 parent 299293c commit c321210

27 files changed

+2892
-309
lines changed

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
dist
22
dist-newstyle
33
.ghc.environment.*
4+
cabal.project.local
45
.cabal-sandbox/
56
cabal.sandbox.config
67
.stack-work/
78
.stack-work-bench/
89
*.yaml.lock
10+
.bench-results
911

1012
*.o
1113
*.hi

‎.travis.yml

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,16 +108,20 @@ install:
108108
- |
109109
echo "packages: ." >> cabal.project
110110
echo "packages: attoparsec-iso8601" >> cabal.project
111+
echo "packages: examples" >> cabal.project
111112
- if [ $HCNUMVER -ge 80200 ] ; then echo 'package aeson' >> cabal.project ; fi
112113
- "if [ $HCNUMVER -ge 80200 ] ; then echo ' ghc-options: -Werror=missing-methods' >> cabal.project ; fi"
113114
- if [ $HCNUMVER -ge 80200 ] ; then echo 'package attoparsec-iso8601' >> cabal.project ; fi
114115
- "if [ $HCNUMVER -ge 80200 ] ; then echo ' ghc-options: -Werror=missing-methods' >> cabal.project ; fi"
116+
- if [ $HCNUMVER -ge 80200 ] ; then echo 'package aeson-examples' >> cabal.project ; fi
117+
- "if [ $HCNUMVER -ge 80200 ] ; then echo ' ghc-options: -Werror=missing-methods' >> cabal.project ; fi"
115118
- |
116-
- "for pkg in $($HCPKG list --simple-output); do echo $pkg | sed 's/-[^-]*$//' | (grep -vE -- '^(aeson|attoparsec-iso8601)$' || true) | sed 's/^/constraints: /' | sed 's/$/ installed/' >> cabal.project.local; done"
119+
- "for pkg in $($HCPKG list --simple-output); do echo $pkg | sed 's/-[^-]*$//' | (grep -vE -- '^(aeson|aeson-examples|attoparsec-iso8601)$' || true) | sed 's/^/constraints: /' | sed 's/$/ installed/' >> cabal.project.local; done"
117120
- cat cabal.project || true
118121
- cat cabal.project.local || true
119122
- if [ -f "./configure.ac" ]; then (cd "." && autoreconf -i); fi
120123
- if [ -f "attoparsec-iso8601/configure.ac" ]; then (cd "attoparsec-iso8601" && autoreconf -i); fi
124+
- if [ -f "examples/configure.ac" ]; then (cd "examples" && autoreconf -i); fi
121125
- ${CABAL} v2-freeze $WITHCOMPILER ${TEST} ${BENCH}
122126
- "cat cabal.project.freeze | sed -E 's/^(constraints: *| *)//' | sed 's/any.//'"
123127
- rm cabal.project.freeze
@@ -134,18 +138,22 @@ script:
134138
- find . -maxdepth 1 -type f -name '*.tar.gz' -exec rm '{}' \;
135139
- PKGDIR_aeson="$(find . -maxdepth 1 -type d -regex '.*/aeson-[0-9.]*')"
136140
- PKGDIR_attoparsec_iso8601="$(find . -maxdepth 1 -type d -regex '.*/attoparsec-iso8601-[0-9.]*')"
141+
- PKGDIR_aeson_examples="$(find . -maxdepth 1 -type d -regex '.*/aeson-examples-[0-9.]*')"
137142
# Generate cabal.project
138143
- rm -rf cabal.project cabal.project.local cabal.project.freeze
139144
- touch cabal.project
140145
- |
141146
echo "packages: ${PKGDIR_aeson}" >> cabal.project
142147
echo "packages: ${PKGDIR_attoparsec_iso8601}" >> cabal.project
148+
echo "packages: ${PKGDIR_aeson_examples}" >> cabal.project
143149
- if [ $HCNUMVER -ge 80200 ] ; then echo 'package aeson' >> cabal.project ; fi
144150
- "if [ $HCNUMVER -ge 80200 ] ; then echo ' ghc-options: -Werror=missing-methods' >> cabal.project ; fi"
145151
- if [ $HCNUMVER -ge 80200 ] ; then echo 'package attoparsec-iso8601' >> cabal.project ; fi
146152
- "if [ $HCNUMVER -ge 80200 ] ; then echo ' ghc-options: -Werror=missing-methods' >> cabal.project ; fi"
153+
- if [ $HCNUMVER -ge 80200 ] ; then echo 'package aeson-examples' >> cabal.project ; fi
154+
- "if [ $HCNUMVER -ge 80200 ] ; then echo ' ghc-options: -Werror=missing-methods' >> cabal.project ; fi"
147155
- |
148-
- "for pkg in $($HCPKG list --simple-output); do echo $pkg | sed 's/-[^-]*$//' | (grep -vE -- '^(aeson|attoparsec-iso8601)$' || true) | sed 's/^/constraints: /' | sed 's/$/ installed/' >> cabal.project.local; done"
156+
- "for pkg in $($HCPKG list --simple-output); do echo $pkg | sed 's/-[^-]*$//' | (grep -vE -- '^(aeson|aeson-examples|attoparsec-iso8601)$' || true) | sed 's/^/constraints: /' | sed 's/$/ installed/' >> cabal.project.local; done"
149157
- cat cabal.project || true
150158
- cat cabal.project.local || true
151159
# Building...
@@ -159,6 +167,7 @@ script:
159167
# HLint..
160168
- if [ $HCNUMVER -ge 80800 ] && [ $HCNUMVER -lt 81000 ] ; then (cd ${PKGDIR_aeson} && hlint -h ${TOP}/.hlint.yaml . attoparsec-iso8601/ pure ffi) ; fi
161169
- if [ $HCNUMVER -ge 80800 ] && [ $HCNUMVER -lt 81000 ] ; then (cd ${PKGDIR_attoparsec_iso8601} && hlint -h ${TOP}/.hlint.yaml .) ; fi
170+
- if [ $HCNUMVER -ge 80800 ] && [ $HCNUMVER -lt 81000 ] ; then (cd ${PKGDIR_aeson_examples} && hlint -h ${TOP}/.hlint.yaml src/) ; fi
162171
# haddock...
163172
- if [ $HCNUMVER -ge 80600 ] ; then ${CABAL} v2-haddock $WITHCOMPILER --with-haddock $HADDOCK ${TEST} ${BENCH} all ; fi
164173
# Building without installed constraints for packages in global-db...

‎CONTRIBUTING.md

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -99,20 +99,35 @@ Of course before submitting a PR, the following steps are recommended:
9999

100100
### Running benchmarks
101101

102-
You need to install `cabal-plan` and `criterion-compare` with
102+
You need to install `cabal-plan`:
103103

104104
```
105-
cabal install cabal-plan criterion-compare
105+
cabal install cabal-plan
106106
```
107107
108108
Then to build benchmarks we use a different project, which builds
109109
`aeson` as a package with a different name to avoid rebuilding `criterion`
110-
etc tools all the time:
110+
etc tools all the time. There is a helper script which usage
111+
can be as simple as:
111112
112113
```
113-
cabal build --project-file cabal.bench.project all
114+
git checkout master
115+
./bench.sh run -n master
116+
git checkout your-branch
117+
./bench.sh run -n your-branch
118+
./bench.sh compare master your-branch
114119
```
115120
116-
Then to run benchmarks
121+
which will output a table like
117122
118-
... to be completed
123+
```
124+
Benchmark master your-branch
125+
Examples/decode/github-issues/lazy 1.77e-3 1.76e-3 -0.68%
126+
Examples/decode/github-issues/strict 1.75e-3 1.69e-3 -3.29%
127+
Examples/decode/jp100/lazy 1.97e-3 1.98e-3 +0.43%
128+
Examples/decode/jp100/strict 1.94e-3 1.96e-3 +1.10%
129+
Examples/decode/twitter100/lazy 1.54e-3 1.59e-3 +2.98%
130+
Examples/decode/twitter100/strict 1.51e-3 1.51e-3 -0.20%
131+
```
132+
133+
Run `./bench.sh help` for more details.

‎LICENSE

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
Copyright (c) 2011, MailRank, Inc.
1+
Copyright (c) 2011, MailRank, Inc. 2014-2020 Aeson project contributors
22

33
All rights reserved.
44

‎aeson.cabal

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,6 @@ extra-source-files:
3535
benchmarks/json-data/*.json
3636
cbits/*.c
3737
changelog.md
38-
examples/*.cabal
39-
examples/*.hs
40-
examples/Twitter/*.hs
4138
ffi/Data/Aeson/Parser/*.hs
4239
include/*.h
4340
tests/JSONTestSuite/test_parsing/*.json

‎bench.sh

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
#!/bin/sh
2+
3+
# Aeson Benchmark Runner
4+
#######################################################################
5+
6+
# Configuration
7+
#######################################################################
8+
9+
HC=ghc
10+
SAVENAME="auto-$(git rev-parse --short HEAD)"
11+
CONFIDENCE=0.99
12+
RUNS=""
13+
PATTERN=""
14+
15+
AESON_BENCH_DATADIR="benchmarks/json-data"
16+
export AESON_BENCH_DATADIR
17+
18+
# usage
19+
#######################################################################
20+
21+
usage() {
22+
cat <<EOF
23+
./bench.sh - Aeson benchmark runner
24+
25+
Usage: ./bench.sh [help|run|compare] [options]
26+
27+
help - print this message
28+
-------------------------
29+
30+
./bench.sh help
31+
32+
run - run a benchmark suite
33+
---------------------------
34+
35+
./bench.sh run [options]
36+
37+
Options:
38+
39+
-w HC, --with-compiler HC Compiler to use
40+
-n NAME, --name NAME Name to save benchmark results as (current: $SAVENAME)
41+
-p PATTERN, --pattern PATTERN Which benchmarks to run
42+
--confidence CI Confidence interval (default: $CONFIDENCE)
43+
44+
build - only build a benchmark suite
45+
-----------------------------------
46+
47+
./bench.sh build [options]
48+
49+
Options:
50+
51+
-w HC, --with-compiler HC Compiler to use
52+
EOF
53+
}
54+
55+
# "library"
56+
#######################################################################
57+
58+
RED='\033[0;31m'
59+
GREEN='\033[0;32m'
60+
BLUE='\033[0;34m'
61+
CYAN='\033[0;96m'
62+
RESET='\033[0m' # No Color
63+
64+
putError() {
65+
echo "${RED}ERROR:${RESET} $*"
66+
}
67+
68+
putInfo() {
69+
echo "${CYAN}INFO:${RESET} $*"
70+
}
71+
72+
command() {
73+
echo "${BLUE}RUN:${RESET} $*"
74+
"$@"
75+
exitcode=$?
76+
if [ $exitcode -ne 0 ]; then
77+
echo "${RED}FAIL:${RESET} $*"
78+
exit 1
79+
fi
80+
}
81+
82+
# build
83+
#######################################################################
84+
85+
cmdBuild() {
86+
# Argument parsing
87+
88+
while [ $# -gt 0 ]; do
89+
arg=$1
90+
case $arg in
91+
-w|--with-compiler)
92+
HC=$2
93+
shift
94+
shift
95+
;;
96+
*)
97+
putError "Unknown argument '$1'"
98+
usage
99+
exit 1
100+
;;
101+
esac
102+
done
103+
104+
# Checking existence of tools
105+
putInfo "Checking tools"
106+
command cabal --version
107+
108+
# Building
109+
putInfo "Building benchmarks"
110+
111+
command cabal build --project-file cabal.bench.project -w "$HC" aeson-benchmark-suite criterion-compare-txt
112+
}
113+
114+
# run
115+
#######################################################################
116+
117+
cmdRun() {
118+
# Argument parsing
119+
120+
while [ $# -gt 0 ]; do
121+
arg=$1
122+
case $arg in
123+
-w|--with-compiler)
124+
HC="$2"
125+
shift
126+
shift
127+
;;
128+
-n|--name)
129+
SAVENAME="$2"
130+
shift
131+
shift
132+
;;
133+
-p|--pattern)
134+
PATTERN="$2"
135+
shift
136+
shift
137+
;;
138+
--confidence)
139+
CONFIDENCE="$2"
140+
shift
141+
shift
142+
;;
143+
*)
144+
putError "Unknown argument '$1'"
145+
usage
146+
exit 1
147+
;;
148+
esac
149+
done
150+
151+
CSV=".bench-results/${SAVENAME}.csv"
152+
if [ -f "$CSV" ]; then
153+
putError "$CSV already exists, aborting..."
154+
exit 1
155+
fi
156+
157+
# Checking if the target file exits
158+
159+
# Checking existence of tools
160+
putInfo "Checking tools"
161+
command cabal --version
162+
command cabal-plan --version
163+
164+
# Building
165+
putInfo "Building benchmarks"
166+
command cabal build --project-file cabal.bench.project -w "$HC" aeson-benchmark-suite
167+
168+
# Running
169+
putInfo "Running benchmarks: $SAVENAME"
170+
command mkdir -p .bench-results
171+
command "$(cabal-plan list-bin aeson-benchmark-suite)" --csv "$CSV" --ci "$CONFIDENCE" -m pattern "$PATTERN"
172+
}
173+
174+
# compare
175+
#######################################################################
176+
177+
cmdCompare() {
178+
# Argument parsing
179+
180+
while [ $# -gt 0 ]; do
181+
arg=$1
182+
case $arg in
183+
-w|--with-compiler)
184+
HC=$2
185+
shift
186+
shift
187+
;;
188+
-n|--name)
189+
SAVENAME=$2
190+
shift
191+
shift
192+
;;
193+
*)
194+
RUNS="$RUNS $1"
195+
shift
196+
;;
197+
esac
198+
done
199+
200+
# Building
201+
putInfo "Building criterion-compare-txt"
202+
command cabal build --project-file cabal.bench.project -w "$HC" criterion-compare-txt
203+
204+
# Comparing th eresults
205+
putInfo "Comparing runs:$RUNS"
206+
207+
# Map runs to CVS files
208+
CSV=""
209+
for run in $RUNS; do
210+
CSV="$CSV .bench-results/${run}.csv"
211+
done
212+
213+
# shellcheck disable=SC2086
214+
command "$(cabal-plan list-bin criterion-compare-txt)" $CSV
215+
}
216+
217+
# main
218+
#######################################################################
219+
220+
if [ $# -le 0 ]; then
221+
usage
222+
exit 1
223+
fi
224+
225+
case $1 in
226+
run)
227+
shift;
228+
cmdRun "$@"
229+
;;
230+
compare)
231+
shift;
232+
cmdCompare "$@"
233+
;;
234+
build)
235+
shift;
236+
cmdBuild "$@"
237+
;;
238+
help)
239+
usage
240+
;;
241+
*)
242+
putError "Unknown command '$1'"
243+
usage
244+
exit 1
245+
;;
246+
esac

0 commit comments

Comments
 (0)