Skip to content

Commit 0260c0e

Browse files
luanpotterluanpotter
authored
perf: Add GitHub Action to test benchmark on PRs and second benchmark (#49)
* perf: Add GitHub Action to test benchmark on PRs * Add second benchmark file * Add comprehensive benchmark
1 parent 20d4ebe commit 0260c0e

File tree

4 files changed

+317
-38
lines changed

4 files changed

+317
-38
lines changed

‎.github/workflows/benchmark.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
name: Benchmark
2+
3+
on: [pull_request]
4+
5+
jobs:
6+
benchmark:
7+
runs-on: ubuntu-latest
8+
steps:
9+
- uses: actions/checkout@v3
10+
11+
- uses: dart-lang/setup-dart@v1
12+
- run: dart pub get
13+
14+
- uses: luanpotter/dart-benchmark-action@v0.1.10
15+
with:
16+
paths: "."
17+
env:
18+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎benchmark/comprehensive_benchmark.dart‎

Lines changed: 258 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,258 @@
1+
import 'dart:math';
2+
3+
import 'package:benchmark_harness/benchmark_harness.dart';
4+
import 'package:ordered_set/comparing.dart';
5+
import 'package:ordered_set/ordered_set.dart';
6+
7+
const _maxStartingOperations = 1000;
8+
const _maxElement = 1000;
9+
10+
class ComprehensiveBenchmark extends BenchmarkBase {
11+
final Random r;
12+
final _runtimes = <_Runtime>[];
13+
14+
ComprehensiveBenchmark({
15+
required int seed,
16+
}) : r = Random(seed),
17+
super('Comprehensive Benchmark');
18+
19+
static void main() {
20+
ComprehensiveBenchmark(seed: 69420).report();
21+
}
22+
23+
@override
24+
void setup() {
25+
final primes = [2, 3, 5, 7, 11];
26+
_runtimes.clear();
27+
_runtimes.addAll(
28+
[
29+
// all elements have the same compare factor
30+
Comparing.on<int>((e) => 0),
31+
// all elements are only equal to themselves
32+
Comparing.on<int>((e) => e),
33+
// equal by certain prime factor count
34+
...primes.map(
35+
(p) => Comparing.on<int>((e) => _countFactors(e, p)),
36+
),
37+
].map((e) => _Runtime(r: r, compare: e)),
38+
);
39+
}
40+
41+
@override
42+
void exercise() {
43+
for (final runtime in _runtimes) {
44+
runtime.clear();
45+
runtime.iterate();
46+
}
47+
}
48+
}
49+
50+
class _Runtime {
51+
final Random r;
52+
53+
var _totalOperations = 0;
54+
final List<_Operation> _queue;
55+
final OrderedSet<int> _set;
56+
57+
_Runtime({
58+
required this.r,
59+
required Comparator<int> compare,
60+
}) : _set = OrderedSet<int>(compare),
61+
_queue = [];
62+
63+
void clear() {
64+
_totalOperations = 0;
65+
_queue.clear();
66+
_set.clear();
67+
}
68+
69+
void iterate() {
70+
while (_totalOperations == 0 || _queue.isNotEmpty) {
71+
if (_totalOperations < _maxStartingOperations) {
72+
for (var i = 0; i < r.nextInt(3) + 2; i++) {
73+
_queueOp(_randomOperation());
74+
}
75+
}
76+
77+
final op = _queue.removeAt(0);
78+
op.execute(this, _set).forEach(_queueOp);
79+
}
80+
}
81+
82+
void _queueOp(_Operation op) {
83+
_totalOperations++;
84+
_queue.insert(r.nextInt(_queue.length + 1), op);
85+
}
86+
87+
_Operation _randomOperation() {
88+
final type = _OperationType.values[r.nextInt(_OperationType.values.length)];
89+
switch (type) {
90+
case _OperationType.add:
91+
return _AddOperation(_randomElement());
92+
case _OperationType.removeIdx:
93+
if (_set.isEmpty) {
94+
return _AddOperation(_randomElement());
95+
}
96+
return _RemoveIdxOperation(r.nextInt(_set.length));
97+
case _OperationType.removeElement:
98+
if (_set.isEmpty) {
99+
return _AddOperation(_randomElement());
100+
}
101+
return _RemoveElementOperation(_set.elementAt(r.nextInt(_set.length)));
102+
case _OperationType.removeWhere:
103+
return _RemoveWhereOperation(_randomElement());
104+
case _OperationType.visit:
105+
return _VisitOperation(_randomElement());
106+
case _OperationType.iterateThenAdd:
107+
return _IterateThenAddOperation(_randomElement());
108+
case _OperationType.iterateThenRemove:
109+
return _IterateThenRemoveOperation(_randomElement());
110+
}
111+
}
112+
113+
int _randomElement() => r.nextInt(_maxElement) + 1;
114+
}
115+
116+
enum _OperationType {
117+
// when queued, generates a random element; then adds using `add`
118+
add,
119+
// when queued, selects a random index; then removes using `removeAt`
120+
removeIdx,
121+
// when queued, selects a random element; then removes using `remove`
122+
removeElement,
123+
// when queued, generates a random factor; then removes all elements with
124+
// that factor using `removeWhere`
125+
removeWhere,
126+
// when queued, generates a random factor; then finds the elements matching
127+
// that factor, using normal for iteration
128+
visit,
129+
// when queued, generates two random factors; iterates over the set,
130+
//finds elements that match the first factor, then multiplies them by
131+
//the second factor, queue adding the results with the `add` operation
132+
iterateThenAdd,
133+
// when queued, generates a random factor; iterates over the set, finding
134+
// elements that match the factor, then queue their removal with
135+
// the `removeElement` operation
136+
iterateThenRemove,
137+
}
138+
139+
abstract class _Operation {
140+
final _OperationType type;
141+
142+
const _Operation(this.type);
143+
144+
List<_Operation> execute(_Runtime runtime, OrderedSet<int> set);
145+
}
146+
147+
class _AddOperation extends _Operation {
148+
final int element;
149+
150+
_AddOperation(this.element) : super(_OperationType.add);
151+
152+
@override
153+
List<_Operation> execute(_Runtime runtime, OrderedSet<int> set) {
154+
set.add(element);
155+
return [];
156+
}
157+
}
158+
159+
class _RemoveIdxOperation extends _Operation {
160+
final int index;
161+
162+
_RemoveIdxOperation(this.index) : super(_OperationType.removeIdx);
163+
164+
@override
165+
List<_Operation> execute(_Runtime runtime, OrderedSet<int> set) {
166+
if (index < set.length) {
167+
set.removeAt(index);
168+
}
169+
return [];
170+
}
171+
}
172+
173+
class _RemoveElementOperation extends _Operation {
174+
final int element;
175+
176+
_RemoveElementOperation(this.element) : super(_OperationType.removeElement);
177+
178+
@override
179+
List<_Operation> execute(_Runtime runtime, OrderedSet<int> set) {
180+
set.remove(element);
181+
return [];
182+
}
183+
}
184+
185+
class _RemoveWhereOperation extends _Operation {
186+
final int factor;
187+
188+
_RemoveWhereOperation(this.factor) : super(_OperationType.removeWhere);
189+
190+
@override
191+
List<_Operation> execute(_Runtime runtime, OrderedSet<int> set) {
192+
set.removeWhere((e) => e % factor == 0);
193+
return [];
194+
}
195+
}
196+
197+
class _VisitOperation extends _Operation {
198+
final int factor;
199+
200+
_VisitOperation(this.factor) : super(_OperationType.visit);
201+
202+
@override
203+
List<_Operation> execute(_Runtime runtime, OrderedSet<int> set) {
204+
final output = <_Operation>[];
205+
for (final e in set) {
206+
if (e % factor == 0) {
207+
output.add(_AddOperation(e * factor));
208+
}
209+
}
210+
return output;
211+
}
212+
}
213+
214+
class _IterateThenAddOperation extends _Operation {
215+
final int factor;
216+
217+
_IterateThenAddOperation(this.factor) : super(_OperationType.iterateThenAdd);
218+
219+
@override
220+
List<_Operation> execute(_Runtime runtime, OrderedSet<int> set) {
221+
final toAdd = <int>[];
222+
for (final e in set) {
223+
if (e % factor == 0) {
224+
toAdd.add(e);
225+
}
226+
}
227+
228+
return toAdd.map(_AddOperation.new).toList();
229+
}
230+
}
231+
232+
class _IterateThenRemoveOperation extends _Operation {
233+
final int factor;
234+
235+
_IterateThenRemoveOperation(this.factor)
236+
: super(_OperationType.iterateThenRemove);
237+
238+
@override
239+
List<_Operation> execute(_Runtime runtime, OrderedSet<int> set) {
240+
final toRemove = <int>[];
241+
for (final e in set) {
242+
if (e % factor == 0) {
243+
toRemove.add(e);
244+
}
245+
}
246+
return toRemove.map(_RemoveElementOperation.new).toList();
247+
}
248+
}
249+
250+
int _countFactors(int initialValue, int factor) {
251+
var count = 0;
252+
var value = initialValue;
253+
while (value % factor == 0) {
254+
count++;
255+
value ~/= factor;
256+
}
257+
return count;
258+
}

‎benchmark/iteration_benchmark.dart‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
import 'dart:math';
2+
3+
import 'package:benchmark_harness/benchmark_harness.dart';
4+
import 'package:ordered_set/ordered_set.dart';
5+
6+
import '../test/comparable_object.dart';
7+
8+
class IterationBenchmark extends BenchmarkBase {
9+
late final OrderedSet<ComparableObject> set;
10+
11+
IterationBenchmark() : super('Iteration Benchmark');
12+
13+
static void main() {
14+
IterationBenchmark().report();
15+
}
16+
17+
@override
18+
void setup() {
19+
set = OrderedSet();
20+
for (var i = 0; i < 1000; i++) {
21+
final l = (10 + sqrt(i)).floor();
22+
for (var j = 0; j <= l; j++) {
23+
set.add(ComparableObject(i, '$i-$j'));
24+
}
25+
}
26+
}
27+
28+
@override
29+
void exercise() {
30+
for (final element in set) {
31+
_consume(element);
32+
}
33+
}
34+
35+
void _consume(ComparableObject obj) {
36+
// NO-OP
37+
}
38+
}

‎benchmark/main.dart‎

Lines changed: 3 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,42 +1,7 @@
1-
import 'dart:math';
2-
3-
import 'package:benchmark_harness/benchmark_harness.dart';
4-
import 'package:ordered_set/ordered_set.dart';
5-
6-
import '../test/comparable_object.dart';
7-
8-
class IterationBenchmark extends BenchmarkBase {
9-
late final OrderedSet<ComparableObject> set;
10-
11-
IterationBenchmark() : super('Template');
12-
13-
static void main() {
14-
IterationBenchmark().report();
15-
}
16-
17-
@override
18-
void setup() {
19-
set = OrderedSet();
20-
for (var i = 0; i < 1000; i++) {
21-
final l = (10 + sqrt(i)).floor();
22-
for (var j = 0; j <= l; j++) {
23-
set.add(ComparableObject(i, '$i-$j'));
24-
}
25-
}
26-
}
27-
28-
@override
29-
void exercise() {
30-
for (final element in set) {
31-
_consume(element);
32-
}
33-
}
34-
35-
void _consume(ComparableObject obj) {
36-
// NO-OP
37-
}
38-
}
1+
import 'comprehensive_benchmark.dart';
2+
import 'iteration_benchmark.dart';
393

404
void main() {
415
IterationBenchmark.main();
6+
ComprehensiveBenchmark.main();
427
}

0 commit comments

Comments
 (0)