Skip to content

Commit a98011a

Browse files
jensjohaCommit Queue
jensjoha
authored and
Commit Queue
committed
[analyzer] Initial LSP server e2e benchmark
A benchmark that launches the dart analyzer server in language server protocol mode and communicates with is as an IDE. In this case the benchmark generates between 16 and 1024 copies of the abstract scanner (to get a large amount of code) as well as imports and exports all the files in cycles and/or chains; performs an edit, requests completion and times initial startup (with no cache), completion after change and when it's done analyzing after the change. It does this in several modes than change the way the files are imported and exported: * ImportCycle where file1 imports file2 etc and the last file imports file1. There are no exports. * ImportChain where file1 imports file2 etc and the last file doesn't import anything. There are no exports. * ImportExportCycle where file1 imports and exports file2 etc and the last file imports and exports file1. * ImportExportChain where file1 imports and exports file2 etc and the last file doesn't import or export anything. * ImportCycleExportChain where file1 imports and exports file2 etc and the last file imports file1 but doesn't export anything. For ImportCycle, ImportChain and ImportExportChain things appear to scale ~linear and - using AOT - have timeings in this ballpark (this is specifically for ImportCycle): +------+-----------+------------+------------+ | Size | Initial | Completion | Fully done | +------+-----------+------------+------------+ | 16 | 0.46561 | 0.158765 | 0.40474 | | 32 | 0.901167 | 0.268819 | 0.859874 | | 64 | 1.657207 | 0.428747 | 1.488365 | | 128 | 3.178606 | 0.843576 | 3.040237 | | 256 | 6.015557 | 1.737661 | 6.010487 | | 512 | 12.08567 | 2.979242 | 11.736878 | | 1024 | 24.273368 | 6.101671 | 24.018495 | +------+-----------+------------+------------+ For ImportExportCycle and ImportCycleExportChain it scales worse and e.g. ImportExportCycle looks like this: +------+-----------+------------+------------+ | Size | Initial | Completion | Fully done | +------+-----------+------------+------------+ | 16 | 0.46673 | 0.169486 | 0.406448 | | 32 | 0.875871 | 0.242876 | 0.85543 | | 64 | 1.583077 | 0.465915 | 1.506953 | | 128 | 3.198071 | 0.903894 | 3.09165 | | 256 | 6.786677 | 2.149489 | 6.779569 | | 512 | 17.346131 | 8.92149 | 17.971033 | | 1024 | 63.358453 | 46.152089 | 65.401559 | +------+-----------+------------+------------+ (In the tables 'Completion' is time until completion answers after a top-level change and 'Fully done' is time until the analyzer stops analyzing after a top-level change). Change-Id: Id7214c0d6c14199f39c0c8a6a8b4941a0e575dc3 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/413401 Reviewed-by: Brian Wilkerson <[email protected]> Commit-Queue: Jens Johansen <[email protected]>
1 parent 219471c commit a98011a

File tree

16 files changed

+6795
-0
lines changed

16 files changed

+6795
-0
lines changed

‎pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/big_chain_benchmark.dart‎

Lines changed: 272 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,272 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import 'dart:io';
6+
7+
import '../lsp_benchmark.dart';
8+
import '../messages.dart';
9+
10+
Future<void> main() async {
11+
StringBuffer sb = StringBuffer();
12+
for (CodeType codeType in CodeType.values) {
13+
for (int numFiles in [16, 32, 64, 128, 256, 512, 1024]) {
14+
Directory tmpDir = Directory.systemTemp.createTempSync('lsp_benchmark');
15+
try {
16+
Directory cacheDir = Directory.fromUri(tmpDir.uri.resolve('cache/'))
17+
..createSync(recursive: true);
18+
Directory dartDir = Directory.fromUri(tmpDir.uri.resolve('dart/'))
19+
..createSync(recursive: true);
20+
copyData(dartDir.uri, numFiles, codeType);
21+
BigChainBenchmark benchmark = BigChainBenchmark(
22+
dartDir.uri,
23+
cacheDir.uri,
24+
numFiles: numFiles,
25+
);
26+
try {
27+
await benchmark.run();
28+
} finally {
29+
benchmark.exit();
30+
}
31+
32+
print('====================');
33+
print('$numFiles files / $codeType:');
34+
print(
35+
'Initial: '
36+
'${formatDuration(benchmark.firstAnalyzingDuration)}',
37+
);
38+
print(
39+
'Completion after change: '
40+
'${formatDuration(benchmark.completionAfterChange)}',
41+
);
42+
print(
43+
'Fully done after change: '
44+
'${formatDuration(benchmark.doneAfterChange)}',
45+
);
46+
print('====================');
47+
sb.writeln('$numFiles files / $codeType:');
48+
sb.writeln(
49+
'Initial: '
50+
'${formatDuration(benchmark.firstAnalyzingDuration)}',
51+
);
52+
sb.writeln(
53+
'Completion after change: '
54+
'${formatDuration(benchmark.completionAfterChange)}',
55+
);
56+
sb.writeln(
57+
'Fully done after change: '
58+
'${formatDuration(benchmark.doneAfterChange)}',
59+
);
60+
sb.writeln();
61+
} finally {
62+
tmpDir.deleteSync(recursive: true);
63+
}
64+
}
65+
}
66+
67+
print('==================================');
68+
print(sb.toString().trim());
69+
print('==================================');
70+
}
71+
72+
void copyData(Uri tmp, int numFiles, CodeType copyType) {
73+
Uri filesUri = Platform.script.resolve('files/');
74+
Uri tmpLib = tmp.resolve('lib/');
75+
Directory.fromUri(tmpLib).createSync();
76+
Directory files = Directory.fromUri(filesUri);
77+
for (var file in files.listSync()) {
78+
if (file is! File) continue;
79+
String filename = file.uri.pathSegments.last;
80+
file.copySync(tmpLib.resolve(filename).toFilePath());
81+
}
82+
File copyMe = File.fromUri(filesUri.resolve('copy_me/copy_me.dart'));
83+
String copyMeData = copyMe.readAsStringSync();
84+
Uri copyToDir = tmpLib.resolve('copies/');
85+
Directory.fromUri(copyToDir).createSync(recursive: true);
86+
87+
for (int i = 1; i <= numFiles; i++) {
88+
String nextFile = getFilenameFor(i == numFiles ? 1 : i + 1);
89+
String import = "import '$nextFile' as nextFile;";
90+
String export = "export '$nextFile';";
91+
switch (copyType) {
92+
case CodeType.ImportCycle:
93+
export = '';
94+
case CodeType.ImportChain:
95+
export = '';
96+
if (i == numFiles) {
97+
import = '';
98+
}
99+
case CodeType.ImportExportChain:
100+
if (i == numFiles) {
101+
import = '';
102+
export = '';
103+
}
104+
case CodeType.ImportCycleExportChain:
105+
if (i == numFiles) {
106+
export = '';
107+
}
108+
case CodeType.ImportExportCycle:
109+
// As default values.
110+
}
111+
112+
String fooMethod;
113+
if (import.isEmpty) {
114+
fooMethod = '''
115+
String foo(int i) {
116+
if (i == 0) return "foo";
117+
return "bar";
118+
}''';
119+
} else {
120+
fooMethod = '''
121+
String foo(int i) {
122+
if (i == 0) return "foo";
123+
return nextFile.foo(i-1);
124+
}''';
125+
}
126+
File.fromUri(copyToDir.resolve(getFilenameFor(i))).writeAsStringSync('''
127+
$import
128+
$export
129+
130+
$copyMeData
131+
132+
$fooMethod
133+
134+
String get$i() {
135+
return "$i";
136+
}
137+
138+
''');
139+
}
140+
141+
File.fromUri(copyToDir.resolve('main.dart')).writeAsStringSync("""
142+
import '${getFilenameFor(1)}';
143+
144+
void main(List<String> arguments) {
145+
146+
}
147+
""");
148+
}
149+
150+
String formatDuration(Duration? duration) {
151+
if (duration == null) return 'N/A';
152+
int seconds = duration.inSeconds;
153+
int ms = duration.inMicroseconds - seconds * Duration.microsecondsPerSecond;
154+
return '$seconds.${ms.toString().padLeft(6, '0')}';
155+
}
156+
157+
String getFilenameFor(int i) {
158+
return "file${i.toString().padLeft(5, '0')}.dart";
159+
}
160+
161+
class BigChainBenchmark extends LspBenchmark {
162+
@override
163+
final Uri rootUri;
164+
@override
165+
final Uri cacheFolder;
166+
167+
Duration? completionAfterChange;
168+
Duration? doneAfterChange;
169+
170+
int numFiles;
171+
172+
BigChainBenchmark(this.rootUri, this.cacheFolder, {required this.numFiles});
173+
174+
@override
175+
LaunchFrom get launchFrom => LaunchFrom.Dart;
176+
177+
@override
178+
Future<void> afterInitialization() async {
179+
Uri tmpLib = rootUri.resolve('lib/');
180+
Uri lastFileUri = tmpLib.resolve('copies/${getFilenameFor(numFiles)}');
181+
Uri mainFileUri = tmpLib.resolve('copies/main.dart');
182+
var mainFileContent = File.fromUri(mainFileUri).readAsStringSync();
183+
var lastFileContent = File.fromUri(lastFileUri).readAsStringSync();
184+
var lastFileContentLines = lastFileContent.split('\n');
185+
186+
Future<void> openFile(Uri uri, String content) async {
187+
await send(Messages.open(uri, 1, content));
188+
await (await send(
189+
Messages.documentColor(uri, largestIdSeen + 1),
190+
))?.completer.future;
191+
await (await send(
192+
Messages.documentSymbol(lastFileUri, largestIdSeen + 1),
193+
))?.completer.future;
194+
195+
// TODO(jensj): Possibly send this - as the IDE does - too?
196+
// textDocument/semanticTokens/full
197+
// textDocument/codeAction
198+
// textDocument/documentLink
199+
// textDocument/codeAction
200+
// textDocument/semanticTokens/range
201+
// textDocument/inlayHint
202+
// textDocument/foldingRange
203+
// textDocument/codeAction
204+
// textDocument/documentHighlight
205+
// textDocument/codeAction
206+
// textDocument/codeLens
207+
// textDocument/codeAction
208+
}
209+
210+
// Open main file.
211+
await openFile(mainFileUri, mainFileContent);
212+
213+
// Open last file.
214+
await openFile(lastFileUri, lastFileContent);
215+
216+
// Change last file: Add a top-level method.
217+
await send(
218+
Messages.didChange(
219+
lastFileUri,
220+
version: 2,
221+
insertAtLine: lastFileContentLines.length - 1 /* line 0-indexed */,
222+
insert: '\nString bar() {\n return "bar";\n}',
223+
),
224+
);
225+
226+
// Request the symbols (although we will ignore the response which we won't
227+
// await).
228+
await send(Messages.documentSymbol(lastFileUri, largestIdSeen + 1));
229+
230+
// Start typing in the main file and request auto-completion.
231+
await send(
232+
Messages.didChange(
233+
mainFileUri,
234+
version: 2,
235+
insertAtLine: 3 /* line 0-indexed; at blank line inside main */,
236+
insertAtCharacter: 2,
237+
insert: 'ge',
238+
),
239+
);
240+
Future<Map<String, dynamic>> completionFuture =
241+
(await send(
242+
Messages.completion(
243+
mainFileUri,
244+
largestIdSeen + 1,
245+
line: 3,
246+
character: 4 /* after the 'ge' just typed */,
247+
),
248+
))!.completer.future;
249+
250+
Stopwatch stopwatch = Stopwatch()..start();
251+
var completionResponse = await completionFuture;
252+
List<dynamic> completionItems =
253+
completionResponse['result']['items'] as List;
254+
completionAfterChange = stopwatch.elapsed;
255+
print(
256+
'Got ${completionItems.length} completion items '
257+
'in $completionAfterChange',
258+
);
259+
await waitWhileAnalyzing();
260+
stopwatch.stop();
261+
doneAfterChange = stopwatch.elapsed;
262+
print('Fully done after $doneAfterChange');
263+
}
264+
}
265+
266+
enum CodeType {
267+
ImportCycle,
268+
ImportChain,
269+
ImportExportCycle,
270+
ImportExportChain,
271+
ImportCycleExportChain,
272+
}

‎pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/files/analysis_options.yaml‎

Whitespace-only changes.

0 commit comments

Comments
 (0)