[analyzer] Benchmark can output json with --json

E.g.

```
out/ReleaseX64/dart-sdk/bin/dart \
  pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/lsp_with_plugin_that_times_out.dart \
  --types=ImportChain \
  --json \
  --files=10,20,30
```

This is preparation for a future CL.

Change-Id: Icba35f3ab2fad6b645109800376c1eb454af07db
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/425501
Reviewed-by: Brian Wilkerson <[email protected]>
Commit-Queue: Jens Johansen <[email protected]>

Author: jensjoha

pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/utils.dart

@@ -2,6 +2,7 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:convert';
 import 'dart:io';
 
 import '../language_server_benchmark.dart';
@@ -233,6 +234,7 @@ Future<void> runHelper(
   bool includePlugin = false,
 }) async {
   int verbosity = 0;
+  bool jsonOutput = false;
   for (String arg in args) {
     if (arg.startsWith('--files=')) {
       numberOfFileOptions =
@@ -249,86 +251,116 @@ Future<void> runHelper(
       }
     } else if (arg.startsWith('--verbosity=')) {
       verbosity = int.parse(arg.substring('--verbosity='.length));
+    } else if (arg == '--json') {
+      jsonOutput = true;
     }
   }
+  if (jsonOutput) {
+    verbosity = -1;
+  }
   StringBuffer sb = StringBuffer();
+  Map<String, int> jsonData = {};
   for (CodeType codeType in codeTypes) {
     for (int numFiles in numberOfFileOptions) {
-      Directory tmpDir = Directory.systemTemp.createTempSync('lsp_benchmark');
       try {
-        Directory cacheDir = Directory.fromUri(tmpDir.uri.resolve('cache/'))
-          ..createSync(recursive: true);
-        Directory dartDir = Directory.fromUri(tmpDir.uri.resolve('dart/'))
-          ..createSync(recursive: true);
-        var runDetails = copyData(
-          dartDir.uri,
-          tmpDir.uri,
-          numFiles,
-          codeType,
-          args,
-          includePlugin: includePlugin,
-        );
-        var benchmark = benchmarkCreator(
-          args,
-          dartDir.uri,
-          cacheDir.uri,
-          runDetails,
-        );
+        Directory tmpDir = Directory.systemTemp.createTempSync('lsp_benchmark');
         try {
-          benchmark.verbosity = verbosity;
-          await benchmark.run();
-        } finally {
-          benchmark.exit();
-        }
-
-        if (verbosity >= 0) print('====================');
-        if (verbosity >= 0) print('$numFiles files / $codeType:');
-        sb.writeln('$numFiles files / $codeType:');
-        for (var durationInfo in benchmark.durationInfo) {
-          if (verbosity >= 0) {
-            print(
-              '${durationInfo.name}: '
-              '${formatDuration(durationInfo.duration)}',
-            );
-          }
-          sb.writeln(
-            '${durationInfo.name}: '
-            '${formatDuration(durationInfo.duration)}',
+          Directory cacheDir = Directory.fromUri(tmpDir.uri.resolve('cache/'))
+            ..createSync(recursive: true);
+          Directory dartDir = Directory.fromUri(tmpDir.uri.resolve('dart/'))
+            ..createSync(recursive: true);
+          var runDetails = copyData(
+            dartDir.uri,
+            tmpDir.uri,
+            numFiles,
+            codeType,
+            args,
+            includePlugin: includePlugin,
           );
-        }
-        for (var memoryInfo in benchmark.memoryInfo) {
-          if (verbosity >= 0) {
-            print(
-              '${memoryInfo.name}: '
-              '${formatKb(memoryInfo.kb)}',
-            );
-          }
-          sb.writeln(
-            '${memoryInfo.name}: '
-            '${formatKb(memoryInfo.kb)}',
+          var benchmark = benchmarkCreator(
+            args,
+            dartDir.uri,
+            cacheDir.uri,
+            runDetails,
           );
-        }
-        if (verbosity >= 0) print('====================');
-        sb.writeln();
-      } finally {
-        try {
-          tmpDir.deleteSync(recursive: true);
-        } catch (e) {
-          // Wait a little and retry.
-          sleep(const Duration(milliseconds: 42));
+          try {
+            benchmark.verbosity = verbosity;
+            await benchmark.run();
+          } finally {
+            benchmark.exit();
+          }
+
+          var caption = '$numFiles files / $codeType';
+          if (jsonOutput) {
+            for (var durationInfo in benchmark.durationInfo) {
+              var key = '$caption: ${durationInfo.name} (ms)';
+              if (jsonData.containsKey(key)) {
+                throw 'Already contains data for $key';
+              }
+              jsonData[key] = durationInfo.duration.inMilliseconds;
+            }
+            for (var memoryInfo in benchmark.memoryInfo) {
+              jsonData['$caption: ${memoryInfo.name} (kb)'] = memoryInfo.kb;
+            }
+          } else {
+            if (verbosity >= 0) print('====================');
+            if (verbosity >= 0) print('$caption:');
+            sb.writeln('$caption:');
+            for (var durationInfo in benchmark.durationInfo) {
+              if (verbosity >= 0) {
+                print(
+                  '${durationInfo.name}: '
+                  '${formatDuration(durationInfo.duration)}',
+                );
+              }
+              sb.writeln(
+                '${durationInfo.name}: '
+                '${formatDuration(durationInfo.duration)}',
+              );
+            }
+            for (var memoryInfo in benchmark.memoryInfo) {
+              if (verbosity >= 0) {
+                print(
+                  '${memoryInfo.name}: '
+                  '${formatKb(memoryInfo.kb)}',
+                );
+              }
+              sb.writeln(
+                '${memoryInfo.name}: '
+                '${formatKb(memoryInfo.kb)}',
+              );
+            }
+            if (verbosity >= 0) print('====================');
+            sb.writeln();
+          }
+        } finally {
           try {
             tmpDir.deleteSync(recursive: true);
           } catch (e) {
-            if (verbosity >= 0) print('Warning: $e');
+            // Wait a little and retry.
+            sleep(const Duration(milliseconds: 42));
+            try {
+              tmpDir.deleteSync(recursive: true);
+            } catch (e) {
+              if (verbosity >= 0) print('Warning: $e');
+            }
           }
         }
+      } catch (e) {
+        stderr.writeln(
+          'Error while processing $numFiles files / $codeType: $e',
+        );
       }
     }
   }
 
-  print('==================================');
-  print(sb.toString().trim());
-  print('==================================');
+  if (jsonOutput) {
+    print(json.encode(jsonData));
+  } else {
+    print('==================================');
+    print(sb.toString().trim());
+    print('==================================');
+  }
 }
 
 enum CodeType {

pkg/analysis_server/tool/benchmark_tools/language_server_benchmark.dart

@@ -17,6 +17,7 @@ abstract class DartLanguageServerBenchmark {
   late final Process p;
   late final Timer longRunningRequestsTimer;
   bool _launched = false;
+  bool _exited = false;
   Completer<bool> _analyzingCompleter = Completer();
   final _buffer = <int>[];
   int? _headerContentLength;
@@ -51,6 +52,7 @@ abstract class DartLanguageServerBenchmark {
   Future<void> afterInitialization();
 
   void exit() {
+    _exited = true;
     longRunningRequestsTimer.cancel();
     if (Platform.isLinux) {
       try {
@@ -304,6 +306,21 @@ abstract class DartLanguageServerBenchmark {
         ]);
     }
 
+    // ignore: unawaited_futures
+    p.exitCode.then((exitCode) {
+      if (verbosity >= 0) {
+        print('Process existed with exitCode $exitCode');
+      }
+      if (!_exited) {
+        _analyzingCompleter.completeError('Process exited 1.');
+        while (_outstandingRequestsWithId.isNotEmpty) {
+          var entry = _outstandingRequestsWithId.entries.first;
+          _outstandingRequestsWithId.remove(entry.key);
+          entry.value.completer.completeError('Process exited 2.');
+        }
+      }
+    });
+
     if (verbosity >= 0) {
       print('Launched with pid ${p.pid}');
     }