Skip to content

Commit 05e04f1

Browse files
polytypicpolytypic
committed
Use a wall time budget for benchmarks and add filtering ability
1 parent 2110a98 commit 05e04f1

17 files changed

+160
-95
lines changed

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ test:
99
@dune runtest --force
1010

1111
bench:
12-
@dune exec --release -- bench/main.exe 10
12+
@dune exec --release -- bench/main.exe -budget 1
1313

1414
clean:
1515
@dune clean

‎bench/barrier.ml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
type t = { counter : int Atomic.t; total : int }
22

3-
let make total = { counter = Atomic.make 0; total }
3+
let make total =
4+
{ counter = Atomic.make 0 |> Multicore_magic.copy_as_padded; total }
5+
|> Multicore_magic.copy_as_padded
46

57
let await { counter; total } =
68
if Atomic.get counter = total then

‎bench/bench.ml‎

Lines changed: 58 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,47 @@
11
module Times = struct
2-
type t = { inverted : bool; times : float array }
2+
type t = { inverted : bool; times : float array; runs : int }
33

4-
let record ~n_domains ?(n_warmups = 3) ?(n_runs = 17) ?(before = Fun.id) ~init
5-
~work ?(after = Fun.id) () =
4+
let record ~n_domains ~budgetf ?(n_warmups = 3) ?(n_runs_min = 7)
5+
?(before = Fun.id) ~init ~work ?(after = Fun.id) () =
6+
let barrier_init = Barrier.make n_domains in
67
let barrier_before = Barrier.make n_domains in
78
let barrier_after = Barrier.make n_domains in
8-
let results = Array.init n_domains @@ fun _ -> Stack.create () in
9+
let results =
10+
Array.init n_domains @@ fun _ ->
11+
Stack.create () |> Multicore_magic.copy_as_padded
12+
in
13+
let budget_used = ref false |> Multicore_magic.copy_as_padded in
14+
let runs = ref 0 |> Multicore_magic.copy_as_padded in
15+
Gc.full_major ();
16+
let budget_start = Mtime_clock.elapsed () in
917
let main domain_i =
1018
for _ = 1 to n_warmups do
11-
if domain_i = 0 then before ();
19+
if domain_i = 0 then begin
20+
before ();
21+
Gc.major ()
22+
end;
1223
let state = init domain_i in
1324
Barrier.await barrier_before;
1425
work domain_i state;
1526
Barrier.await barrier_after;
1627
if domain_i = 0 then after ()
1728
done;
18-
for _run_i = 0 to n_runs - 1 do
19-
if domain_i = 0 then before ();
29+
while !runs < n_runs_min || not !budget_used do
30+
Barrier.await barrier_init;
31+
if domain_i = 0 then begin
32+
before ();
33+
if
34+
let budget_stop = Mtime_clock.elapsed () in
35+
let elapsedf =
36+
Mtime.Span.to_float_ns
37+
(Mtime.Span.abs_diff budget_stop budget_start)
38+
*. (1. /. 1_000_000_000.0)
39+
in
40+
budgetf < elapsedf
41+
then budget_used := true;
42+
incr runs;
43+
Gc.major ()
44+
end;
2045
let state = init domain_i in
2146
Barrier.await barrier_before;
2247
let start = Mtime_clock.elapsed () in
@@ -69,21 +94,32 @@ module Times = struct
6994
times.(run_i) <- times.(run_i) +. Stack.pop results.(domain_i)
7095
done
7196
done;
72-
{ inverted = false; times }
97+
{ inverted = false; times; runs = !runs }
7398

74-
let invert t =
75-
{ inverted = not t.inverted; times = Array.map (fun v -> 1.0 /. v) t.times }
99+
let invert { inverted; times; runs } =
100+
{
101+
inverted = not inverted;
102+
times = Array.map (fun v -> 1.0 /. v) times;
103+
runs;
104+
}
76105
end
77106

78107
module Stats = struct
79-
type t = { mean : float; median : float; sd : float; inverted : bool }
108+
type t = {
109+
mean : float;
110+
median : float;
111+
sd : float;
112+
inverted : bool;
113+
runs : int;
114+
}
80115

81-
let scale factor t =
116+
let scale factor { mean; median; sd; inverted; runs } =
82117
{
83-
mean = t.mean *. factor;
84-
median = t.median *. factor;
85-
sd = t.sd *. factor;
86-
inverted = t.inverted;
118+
mean = mean *. factor;
119+
median = median *. factor;
120+
sd = sd *. factor;
121+
inverted;
122+
runs;
87123
}
88124

89125
let mean_of times =
@@ -99,11 +135,11 @@ module Stats = struct
99135
if n land 1 = 0 then (times.((n asr 1) - 1) +. times.(n asr 1)) /. 2.0
100136
else times.(n asr 1)
101137

102-
let of_times (t : Times.t) =
103-
let mean = mean_of t.times in
104-
let sd = sd_of t.times mean in
105-
let median = median_of t.times in
106-
{ mean; sd; median; inverted = t.inverted }
138+
let of_times Times.{ inverted; times; runs } =
139+
let mean = mean_of times in
140+
let sd = sd_of times mean in
141+
let median = median_of times in
142+
{ mean; sd; median; inverted; runs }
107143

108144
let to_nonbreaking s =
109145
s |> String.split_on_char ' '
@@ -120,6 +156,7 @@ module Stats = struct
120156
if t.inverted then `String "higher-is-better"
121157
else `String "lower-is-better" );
122158
("description", `String description);
159+
("#runs", `Int t.runs);
123160
]
124161
in
125162
[ metric t.median ]

‎bench/bench.mli‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,9 @@ module Times : sig
33

44
val record :
55
n_domains:int ->
6+
budgetf:float ->
67
?n_warmups:int ->
7-
?n_runs:int ->
8+
?n_runs_min:int ->
89
?before:(unit -> unit) ->
910
init:(int -> 's) ->
1011
work:(int -> 's -> unit) ->

‎bench/bench_accumulator.ml‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
open Kcas_data
22
open Bench
33

4-
let run_one ~n_domains ?(factor = 1) ?(n_ops = 60 * factor * Util.iter_factor)
5-
() =
4+
let run_one ~budgetf ~n_domains ?(n_ops = 180 * Util.iter_factor) () =
65
let n_ops = n_ops * n_domains in
76

87
let t = Accumulator.make 0 in
@@ -30,7 +29,7 @@ let run_one ~n_domains ?(factor = 1) ?(n_ops = 60 * factor * Util.iter_factor)
3029

3130
let after () = Atomic.set n_ops_todo n_ops in
3231

33-
let times = Times.record ~n_domains ~init ~work ~after () in
32+
let times = Times.record ~n_domains ~budgetf ~init ~work ~after () in
3433

3534
let name metric =
3635
Printf.sprintf "%s/%d worker%s, 0%% reads" metric n_domains
@@ -53,6 +52,6 @@ let run_one ~n_domains ?(factor = 1) ?(n_ops = 60 * factor * Util.iter_factor)
5352
~units:"M/s";
5453
]
5554

56-
let run_suite ~factor =
55+
let run_suite ~budgetf =
5756
[ 1; 2; 4 ]
58-
|> List.concat_map @@ fun n_domains -> run_one ~n_domains ~factor ()
57+
|> List.concat_map @@ fun n_domains -> run_one ~n_domains ~budgetf ()

‎bench/bench_atomic.ml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ end
1313
type t =
1414
| Op : string * int * 'a * ('a Atomic.t -> unit) * ('a Atomic.t -> unit) -> t
1515

16-
let run_one ?(factor = 1) ?(n_iter = 50 * factor * Util.iter_factor)
16+
let run_one ~budgetf ?(n_iter = 500 * Util.iter_factor)
1717
(Op (name, extra, value, op1, op2)) =
1818
let n_iter = n_iter * extra in
1919

@@ -31,7 +31,7 @@ let run_one ?(factor = 1) ?(n_iter = 50 * factor * Util.iter_factor)
3131
loop n_iter
3232
in
3333

34-
let times = Times.record ~n_domains:1 ~init ~work () in
34+
let times = Times.record ~n_domains:1 ~budgetf ~init ~work () in
3535

3636
List.concat
3737
[
@@ -47,7 +47,7 @@ let run_one ?(factor = 1) ?(n_iter = 50 * factor * Util.iter_factor)
4747
~description:"Number of operations performed over time" ~units:"M/s";
4848
]
4949

50-
let run_suite ~factor =
50+
let run_suite ~budgetf =
5151
[
5252
(let get x = Atomic.get x |> ignore in
5353
Op ("get", 10, 42, get, get));
@@ -65,4 +65,4 @@ let run_suite ~factor =
6565
(let swap x = Atomic.modify x (fun (x, y) -> (y, x)) in
6666
Op ("swap", 2, (4, 2), swap, swap));
6767
]
68-
|> List.concat_map @@ run_one ~factor
68+
|> List.concat_map @@ run_one ~budgetf

‎bench/bench_dllist.ml‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
open Kcas_data
22
open Bench
33

4-
let run_single ?(factor = 1) ?(n_msgs = 10 * factor * Util.iter_factor) () =
4+
let run_single ~budgetf ?(n_msgs = 15 * Util.iter_factor) () =
55
let t = Dllist.create () in
66

77
let init _ = () in
@@ -12,7 +12,7 @@ let run_single ?(factor = 1) ?(n_msgs = 10 * factor * Util.iter_factor) () =
1212
done
1313
in
1414

15-
let times = Times.record ~n_domains:1 ~init ~work () in
15+
let times = Times.record ~n_domains:1 ~budgetf ~init ~work () in
1616

1717
let name metric = Printf.sprintf "%s/single-domain" metric in
1818

@@ -33,8 +33,8 @@ let run_single ?(factor = 1) ?(n_msgs = 10 * factor * Util.iter_factor) () =
3333
~units:"M/s";
3434
]
3535

36-
let run_one ?(n_adders = 2) ?(n_takers = 2) ?(factor = 1)
37-
?(n_msgs = 50 * factor * Util.iter_factor) () =
36+
let run_one ~budgetf ?(n_adders = 2) ?(n_takers = 2) ?(factor = 1)
37+
?(n_msgs = 20 * factor * Util.iter_factor) () =
3838
let n_domains = n_adders + n_takers in
3939

4040
let t = Dllist.create () in
@@ -74,7 +74,7 @@ let run_one ?(n_adders = 2) ?(n_takers = 2) ?(factor = 1)
7474
Atomic.set n_msgs_to_add n_msgs
7575
in
7676

77-
let times = Times.record ~n_domains ~init ~work ~after () in
77+
let times = Times.record ~n_domains ~budgetf ~init ~work ~after () in
7878

7979
let name metric =
8080
let format role blocking n =
@@ -105,8 +105,8 @@ let run_one ?(n_adders = 2) ?(n_takers = 2) ?(factor = 1)
105105
~units:"M/s";
106106
]
107107

108-
let run_suite ~factor =
109-
run_single ~factor ()
108+
let run_suite ~budgetf =
109+
run_single ~budgetf ()
110110
@ (Util.cross [ 1; 2 ] [ 1; 2 ]
111111
|> List.concat_map @@ fun (n_adders, n_takers) ->
112-
run_one ~n_adders ~n_takers ~factor ())
112+
run_one ~budgetf ~n_adders ~n_takers ())

‎bench/bench_hashtbl.ml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ module Int = struct
77
let hash = Fun.id
88
end
99

10-
let run_one ~n_domains ?(factor = 1) ?(n_ops = 50 * factor * Util.iter_factor)
10+
let run_one ~budgetf ~n_domains ?(n_ops = 40 * Util.iter_factor)
1111
?(n_keys = 1000) ~percent_read () =
1212
let t = Hashtbl.create ~hashed_type:(module Int) () in
1313

@@ -48,7 +48,7 @@ let run_one ~n_domains ?(factor = 1) ?(n_ops = 50 * factor * Util.iter_factor)
4848
in
4949
let after () = Atomic.set n_ops_todo n_ops in
5050

51-
let times = Times.record ~n_domains ~init ~work ~after () in
51+
let times = Times.record ~n_domains ~budgetf ~init ~work ~after () in
5252

5353
let name metric =
5454
Printf.sprintf "%s/%d worker%s, %d%% reads" metric n_domains
@@ -73,7 +73,7 @@ let run_one ~n_domains ?(factor = 1) ?(n_ops = 50 * factor * Util.iter_factor)
7373
~units:"M/s";
7474
]
7575

76-
let run_suite ~factor =
76+
let run_suite ~budgetf =
7777
Util.cross [ 90; 50; 10 ] [ 1; 2; 4 ]
7878
|> List.concat_map @@ fun (percent_read, n_domains) ->
79-
run_one ~n_domains ~percent_read ~factor ()
79+
run_one ~budgetf ~n_domains ~percent_read ()

‎bench/bench_loc.ml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ open Bench
33

44
type t = Op : string * int * 'a * ('a Loc.t -> unit) * ('a Loc.t -> unit) -> t
55

6-
let run_one ?(factor = 1) ?(n_iter = 25 * factor * Util.iter_factor)
6+
let run_one ~budgetf ?(n_iter = 250 * Util.iter_factor)
77
(Op (name, extra, value, op1, op2)) =
88
let n_iter = n_iter * extra in
99

@@ -21,7 +21,7 @@ let run_one ?(factor = 1) ?(n_iter = 25 * factor * Util.iter_factor)
2121
loop n_iter
2222
in
2323

24-
let times = Times.record ~n_domains:1 ~init ~work () in
24+
let times = Times.record ~n_domains:1 ~budgetf ~init ~work () in
2525

2626
List.concat
2727
[
@@ -37,7 +37,7 @@ let run_one ?(factor = 1) ?(n_iter = 25 * factor * Util.iter_factor)
3737
~description:"Number of operations performed over time" ~units:"M/s";
3838
]
3939

40-
let run_suite ~factor =
40+
let run_suite ~budgetf =
4141
[
4242
(let get x = Loc.get x |> ignore in
4343
Op ("get", 5, 42, get, get));
@@ -55,4 +55,4 @@ let run_suite ~factor =
5555
(let swap x = Loc.modify x (fun (x, y) -> (y, x)) in
5656
Op ("swap", 1, (4, 2), swap, swap));
5757
]
58-
|> List.concat_map @@ run_one ~factor
58+
|> List.concat_map @@ run_one ~budgetf

‎bench/bench_mvar.ml‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
11
open Kcas_data
22
open Bench
33

4-
let run_one ?(n_adders = 2) ?(blocking_add = false) ?(n_takers = 2)
5-
?(blocking_take = false) ?(factor = 1)
6-
?(n_msgs = 2 * factor * Util.iter_factor) () =
4+
let run_one ~budgetf ?(n_adders = 2) ?(blocking_add = false) ?(n_takers = 2)
5+
?(blocking_take = false) ?(n_msgs = 2 * Util.iter_factor) () =
76
let n_domains = n_adders + n_takers in
87

98
let t = Mvar.create None in
@@ -68,7 +67,7 @@ let run_one ?(n_adders = 2) ?(blocking_add = false) ?(n_takers = 2)
6867
Atomic.set n_msgs_to_add n_msgs
6968
in
7069

71-
let times = Times.record ~n_domains ~init ~work ~after () in
70+
let times = Times.record ~n_domains ~budgetf ~init ~work ~after () in
7271

7372
let name metric =
7473
let format role blocking n =
@@ -99,10 +98,10 @@ let run_one ?(n_adders = 2) ?(blocking_add = false) ?(n_takers = 2)
9998
~units:"M/s";
10099
]
101100

102-
let run_suite ~factor =
101+
let run_suite ~budgetf =
103102
Util.cross
104103
(Util.cross [ 1; 2 ] [ false; true ])
105104
(Util.cross [ 1; 2 ] [ false; true ])
106105
|> List.concat_map
107106
@@ fun ((n_adders, blocking_add), (n_takers, blocking_take)) ->
108-
run_one ~n_adders ~blocking_add ~n_takers ~blocking_take ~factor ()
107+
run_one ~budgetf ~n_adders ~blocking_add ~n_takers ~blocking_take ()

0 commit comments

Comments
 (0)