Skip to content

Commit daf1084

Browse files
jensjohaCommit Queue
jensjoha
authored and
Commit Queue
committed
[analyzer] Add arguments to to benchmarks
This CL adds the option of sending arguments to the benchmark(s) * Specify what dart to use (e.g. `--dart=/path/to/dart-sdk/bin/dart`), e.g. if wanting to run benchmarks on an (old) released version. * Specify the number of files to operate on (e.g. `--files=100,200`). * Specify the code-types to benchmark (e.g. `--types=ImportChain,ImportExportChain`) * Specify verbosity, e.g. to debug (maybe `--verbosity=4`) or to run benchmarks several times only to look at the final result (`--verbosity=-1`). Change-Id: I5c731f3e6480afdff0546a29c599430d2ab6e7be Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/417460 Commit-Queue: Jens Johansen <[email protected]> Reviewed-by: Brian Wilkerson <[email protected]>
1 parent 6659976 commit daf1084

File tree

6 files changed

+123
-52
lines changed

6 files changed

+123
-52
lines changed

‎pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_many_get_fixes_and_get_assists_requests.dart‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,9 @@ import 'utils.dart';
1515
/// Here we'll fire both `edit.getFixes` (seen in reports from users)
1616
/// and `edit.getAssists` (which seems, locally at least, to happen every time
1717
/// the cursor moves).
18-
Future<void> main() async {
18+
Future<void> main(List<String> args) async {
1919
await runHelper(
20+
args,
2021
LegacyManyGetFixesAndGetAssisstRequestsBenchmark.new,
2122
runAsLsp: false,
2223
// The number of files doesn't seem to be important on this one.
@@ -34,6 +35,7 @@ class LegacyManyGetFixesAndGetAssisstRequestsBenchmark
3435
final RunDetails runDetails;
3536

3637
LegacyManyGetFixesAndGetAssisstRequestsBenchmark(
38+
super.args,
3739
this.rootUri,
3840
this.cacheFolder,
3941
this.runDetails,
@@ -97,9 +99,11 @@ class LegacyManyGetFixesAndGetAssisstRequestsBenchmark
9799
durationInfo.add(
98100
DurationInfo('Completion after change', completionAfterChange),
99101
);
100-
print(
101-
'Got ${completionItems.length} completion items '
102-
'in $completionAfterChange',
103-
);
102+
if (verbosity >= 0) {
103+
print(
104+
'Got ${completionItems.length} completion items '
105+
'in $completionAfterChange',
106+
);
107+
}
104108
}
105109
}

‎pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_many_hover_requests.dart‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,9 @@ import 'utils.dart';
1111
/// This benchmark does this 500 times (roughly equal to hovering for 4 seconds)
1212
/// and reports how long it takes before the analysis server is responsive again
1313
/// (measured by when it responds to a completion request).
14-
Future<void> main() async {
14+
Future<void> main(List<String> args) async {
1515
await runHelper(
16+
args,
1617
LegacyManyHoverRequestsBenchmark.new,
1718
runAsLsp: false,
1819
// The number of files doesn't seem to be important on this one.
@@ -29,6 +30,7 @@ class LegacyManyHoverRequestsBenchmark extends DartLanguageServerBenchmark {
2930
final RunDetails runDetails;
3031

3132
LegacyManyHoverRequestsBenchmark(
33+
super.args,
3234
this.rootUri,
3335
this.cacheFolder,
3436
this.runDetails,
@@ -86,9 +88,11 @@ class LegacyManyHoverRequestsBenchmark extends DartLanguageServerBenchmark {
8688
durationInfo.add(
8789
DurationInfo('Completion after change', completionAfterChange),
8890
);
89-
print(
90-
'Got ${completionItems.length} completion items '
91-
'in $completionAfterChange',
92-
);
91+
if (verbosity >= 0) {
92+
print(
93+
'Got ${completionItems.length} completion items '
94+
'in $completionAfterChange',
95+
);
96+
}
9397
}
9498
}

‎pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_typing_temporary_missing_end_brace.dart‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,9 @@ import 'utils.dart';
1616
/// analyzer starts analyzing, then adding `\n \n}`, then requesting completion
1717
/// inside, measuring how long it takes before we get an answer to the
1818
/// completion request.
19-
Future<void> main() async {
19+
Future<void> main(List<String> args) async {
2020
await runHelper(
21+
args,
2122
LegacyTypingTemporaryMissingEndBraceBenchmark.new,
2223
runAsLsp: false,
2324
);
@@ -33,6 +34,7 @@ class LegacyTypingTemporaryMissingEndBraceBenchmark
3334
final RunDetails runDetails;
3435

3536
LegacyTypingTemporaryMissingEndBraceBenchmark(
37+
super.args,
3638
this.rootUri,
3739
this.cacheFolder,
3840
this.runDetails,
@@ -88,7 +90,9 @@ class LegacyTypingTemporaryMissingEndBraceBenchmark
8890
var stopwatch = Stopwatch()..start();
8991
var isNowAnalyzing = await isNowAnalyzingFuture;
9092
if (!isNowAnalyzing) throw 'Unexpectedly switched to not analyzing.';
91-
print('Started analyzing after ${stopwatch.elapsedMilliseconds} ms');
93+
if (verbosity >= 0) {
94+
print('Started analyzing after ${stopwatch.elapsedMilliseconds} ms');
95+
}
9296

9397
// End the brace and type `ge` inside the if.
9498
await send(
@@ -117,9 +121,11 @@ class LegacyTypingTemporaryMissingEndBraceBenchmark
117121
durationInfo.add(
118122
DurationInfo('Completion after change', completionAfterChange),
119123
);
120-
print(
121-
'Got ${completionItems.length} completion items '
122-
'in $completionAfterChange',
123-
);
124+
if (verbosity >= 0) {
125+
print(
126+
'Got ${completionItems.length} completion items '
127+
'in $completionAfterChange',
128+
);
129+
}
124130
}
125131
}

‎pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/lsp_completion_after_change.dart‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ import 'utils.dart';
99
/// Change a file in a big project and reports how long it takes before the
1010
/// analysis server is responsive again (measured by when it responds to a
1111
/// completion request) and when it's actually done analysing.
12-
Future<void> main() async {
13-
await runHelper(LspCompletionAfterChange.new, runAsLsp: true);
12+
Future<void> main(List<String> args) async {
13+
await runHelper(args, LspCompletionAfterChange.new, runAsLsp: true);
1414
}
1515

1616
class LspCompletionAfterChange extends DartLanguageServerBenchmark {
@@ -21,8 +21,12 @@ class LspCompletionAfterChange extends DartLanguageServerBenchmark {
2121

2222
final RunDetails runDetails;
2323

24-
LspCompletionAfterChange(this.rootUri, this.cacheFolder, this.runDetails)
25-
: super(useLspProtocol: true);
24+
LspCompletionAfterChange(
25+
super.args,
26+
this.rootUri,
27+
this.cacheFolder,
28+
this.runDetails,
29+
) : super(useLspProtocol: true);
2630

2731
@override
2832
LaunchFrom get launchFrom => LaunchFrom.Dart;
@@ -116,14 +120,18 @@ class LspCompletionAfterChange extends DartLanguageServerBenchmark {
116120
durationInfo.add(
117121
DurationInfo('Completion after change', completionAfterChange),
118122
);
119-
print(
120-
'Got ${completionItems.length} completion items '
121-
'in $completionAfterChange',
122-
);
123+
if (verbosity >= 0) {
124+
print(
125+
'Got ${completionItems.length} completion items '
126+
'in $completionAfterChange',
127+
);
128+
}
123129
await waitWhileAnalyzing();
124130
stopwatch.stop();
125131
var doneAfterChange = stopwatch.elapsed;
126132
durationInfo.add(DurationInfo('Fully done after change', doneAfterChange));
127-
print('Fully done after $doneAfterChange');
133+
if (verbosity >= 0) {
134+
print('Fully done after $doneAfterChange');
135+
}
128136
}
129137
}

‎pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/utils.dart‎

Lines changed: 46 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -127,17 +127,39 @@ String getFilenameFor(int i) {
127127
}
128128

129129
Future<void> runHelper(
130+
List<String> args,
130131
DartLanguageServerBenchmark Function(
132+
List<String> args,
131133
Uri rootUri,
132134
Uri cacheFolder,
133135
RunDetails runDetails,
134136
)
135137
benchmarkCreator, {
136138
required bool runAsLsp,
137139
List<int> numberOfFileOptions = const [16, 32, 64, 128, 256, 512, 1024],
140+
List<CodeType> codeTypes = CodeType.values,
138141
}) async {
142+
int verbosity = 0;
143+
for (String arg in args) {
144+
if (arg.startsWith('--files=')) {
145+
numberOfFileOptions =
146+
arg.substring('--files='.length).split(',').map(int.parse).toList();
147+
} else if (arg.startsWith('--types=')) {
148+
codeTypes = [];
149+
for (var type in arg.substring('--types='.length).split(',')) {
150+
type = type.toLowerCase();
151+
for (var value in CodeType.values) {
152+
if (value.name.toLowerCase().contains(type)) {
153+
codeTypes.add(value);
154+
}
155+
}
156+
}
157+
} else if (arg.startsWith('--verbosity=')) {
158+
verbosity = int.parse(arg.substring('--verbosity='.length));
159+
}
160+
}
139161
StringBuffer sb = StringBuffer();
140-
for (CodeType codeType in CodeType.values) {
162+
for (CodeType codeType in codeTypes) {
141163
for (int numFiles in numberOfFileOptions) {
142164
Directory tmpDir = Directory.systemTemp.createTempSync('lsp_benchmark');
143165
try {
@@ -146,37 +168,47 @@ Future<void> runHelper(
146168
Directory dartDir = Directory.fromUri(tmpDir.uri.resolve('dart/'))
147169
..createSync(recursive: true);
148170
var runDetails = copyData(dartDir.uri, numFiles, codeType);
149-
var benchmark = benchmarkCreator(dartDir.uri, cacheDir.uri, runDetails);
171+
var benchmark = benchmarkCreator(
172+
args,
173+
dartDir.uri,
174+
cacheDir.uri,
175+
runDetails,
176+
);
150177
try {
178+
benchmark.verbosity = verbosity;
151179
await benchmark.run();
152180
} finally {
153181
benchmark.exit();
154182
}
155183

156-
print('====================');
157-
print('$numFiles files / $codeType:');
184+
if (verbosity >= 0) print('====================');
185+
if (verbosity >= 0) print('$numFiles files / $codeType:');
158186
sb.writeln('$numFiles files / $codeType:');
159187
for (var durationInfo in benchmark.durationInfo) {
160-
print(
161-
'${durationInfo.name}: '
162-
'${formatDuration(durationInfo.duration)}',
163-
);
188+
if (verbosity >= 0) {
189+
print(
190+
'${durationInfo.name}: '
191+
'${formatDuration(durationInfo.duration)}',
192+
);
193+
}
164194
sb.writeln(
165195
'${durationInfo.name}: '
166196
'${formatDuration(durationInfo.duration)}',
167197
);
168198
}
169199
for (var memoryInfo in benchmark.memoryInfo) {
170-
print(
171-
'${memoryInfo.name}: '
172-
'${formatKb(memoryInfo.kb)}',
173-
);
200+
if (verbosity >= 0) {
201+
print(
202+
'${memoryInfo.name}: '
203+
'${formatKb(memoryInfo.kb)}',
204+
);
205+
}
174206
sb.writeln(
175207
'${memoryInfo.name}: '
176208
'${formatKb(memoryInfo.kb)}',
177209
);
178210
}
179-
print('====================');
211+
if (verbosity >= 0) print('====================');
180212
sb.writeln();
181213
} finally {
182214
try {
@@ -187,7 +219,7 @@ Future<void> runHelper(
187219
try {
188220
tmpDir.deleteSync(recursive: true);
189221
} catch (e) {
190-
print('Warning: $e');
222+
if (verbosity >= 0) print('Warning: $e');
191223
}
192224
}
193225
}

0 commit comments

Comments
 (0)