[analyzer] Refactor benchmarks to make it easier to add new benchmarks based on different sources

Change-Id: I7e6f70c62faad55619de6677100608f406091a5e
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/425502
Commit-Queue: Jens Johansen <[email protected]>
Reviewed-by: Brian Wilkerson <[email protected]>

Author: jensjoha

pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/utils.dart renamed to pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/benchmark_utils.dart

@@ -2,20 +2,20 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-import 'dart:convert';
 import 'dart:io';
 
-import '../language_server_benchmark.dart';
 import '../utils.dart';
 
 RunDetails copyData(
   Uri packageDirUri,
   Uri outerDirForAdditionalData,
   int numFiles,
-  CodeType copyType,
+  CodeType? copyType,
   List<String> args, {
-  required bool includePlugin,
+  required ({bool includePlugin})? extraInformation,
 }) {
+  bool includePlugin = extraInformation?.includePlugin ?? false;
+  if (copyType == null) throw 'Unsupported.';
   Uri filesUri = Platform.script.resolve('files/');
   Uri libDirUri = packageDirUri.resolve('lib/');
   Directory.fromUri(libDirUri).createSync();
@@ -201,45 +201,10 @@ analyzer:
   );
 }
 
-String formatDuration(Duration duration) {
-  int seconds = duration.inSeconds;
-  int ms = duration.inMicroseconds - seconds * Duration.microsecondsPerSecond;
-  return '$seconds.${ms.toString().padLeft(6, '0')}';
-}
-
-String formatKb(int kb) {
-  if (kb > 1024) {
-    return '${kb ~/ 1024} MB';
-  } else {
-    return '$kb KB';
-  }
-}
-
-String getFilenameFor(int i) {
-  return "file${i.toString().padLeft(5, '0')}.dart";
-}
-
-Future<void> runHelper(
-  List<String> args,
-  DartLanguageServerBenchmark Function(
-    List<String> args,
-    Uri rootUri,
-    Uri cacheFolder,
-    RunDetails runDetails,
-  )
-  benchmarkCreator, {
-  required bool runAsLsp,
-  List<int> numberOfFileOptions = const [16, 32, 64, 128, 256, 512, 1024],
-  List<CodeType> codeTypes = CodeType.values,
-  bool includePlugin = false,
-}) async {
-  int verbosity = 0;
-  bool jsonOutput = false;
+List<CodeType?> getExtraIterations(List<String> args) {
+  var codeTypes = CodeType.values;
   for (String arg in args) {
-    if (arg.startsWith('--files=')) {
-      numberOfFileOptions =
-          arg.substring('--files='.length).split(',').map(int.parse).toList();
-    } else if (arg.startsWith('--types=')) {
+    if (arg.startsWith('--types=')) {
       codeTypes = [];
       for (var type in arg.substring('--types='.length).split(',')) {
         type = type.toLowerCase();
@@ -249,118 +214,13 @@ Future<void> runHelper(
           }
         }
       }
-    } else if (arg.startsWith('--verbosity=')) {
-      verbosity = int.parse(arg.substring('--verbosity='.length));
-    } else if (arg == '--json') {
-      jsonOutput = true;
-    }
-  }
-  if (jsonOutput) {
-    verbosity = -1;
-  }
-  StringBuffer sb = StringBuffer();
-  Map<String, int> jsonData = {};
-  for (CodeType codeType in codeTypes) {
-    for (int numFiles in numberOfFileOptions) {
-      try {
-        Directory tmpDir = Directory.systemTemp.createTempSync('lsp_benchmark');
-        try {
-          Directory cacheDir = Directory.fromUri(tmpDir.uri.resolve('cache/'))
-            ..createSync(recursive: true);
-          Directory dartDir = Directory.fromUri(tmpDir.uri.resolve('dart/'))
-            ..createSync(recursive: true);
-          var runDetails = copyData(
-            dartDir.uri,
-            tmpDir.uri,
-            numFiles,
-            codeType,
-            args,
-            includePlugin: includePlugin,
-          );
-          var benchmark = benchmarkCreator(
-            args,
-            dartDir.uri,
-            cacheDir.uri,
-            runDetails,
-          );
-          try {
-            benchmark.verbosity = verbosity;
-            await benchmark.run();
-          } finally {
-            benchmark.exit();
-          }
-
-          var caption = '$numFiles files / $codeType';
-          if (jsonOutput) {
-            for (var durationInfo in benchmark.durationInfo) {
-              var key = '$caption: ${durationInfo.name} (ms)';
-              if (jsonData.containsKey(key)) {
-                throw 'Already contains data for $key';
-              }
-              jsonData[key] = durationInfo.duration.inMilliseconds;
-            }
-            for (var memoryInfo in benchmark.memoryInfo) {
-              jsonData['$caption: ${memoryInfo.name} (kb)'] = memoryInfo.kb;
-            }
-          } else {
-            if (verbosity >= 0) print('====================');
-            if (verbosity >= 0) print('$caption:');
-            sb.writeln('$caption:');
-            for (var durationInfo in benchmark.durationInfo) {
-              if (verbosity >= 0) {
-                print(
-                  '${durationInfo.name}: '
-                  '${formatDuration(durationInfo.duration)}',
-                );
-              }
-              sb.writeln(
-                '${durationInfo.name}: '
-                '${formatDuration(durationInfo.duration)}',
-              );
-            }
-            for (var memoryInfo in benchmark.memoryInfo) {
-              if (verbosity >= 0) {
-                print(
-                  '${memoryInfo.name}: '
-                  '${formatKb(memoryInfo.kb)}',
-                );
-              }
-              sb.writeln(
-                '${memoryInfo.name}: '
-                '${formatKb(memoryInfo.kb)}',
-              );
-            }
-            if (verbosity >= 0) print('====================');
-            sb.writeln();
-          }
-        } finally {
-          try {
-            tmpDir.deleteSync(recursive: true);
-          } catch (e) {
-            // Wait a little and retry.
-            sleep(const Duration(milliseconds: 42));
-            try {
-              tmpDir.deleteSync(recursive: true);
-            } catch (e) {
-              if (verbosity >= 0) print('Warning: $e');
-            }
-          }
-        }
-      } catch (e) {
-        stderr.writeln(
-          'Error while processing $numFiles files / $codeType: $e',
-        );
-      }
     }
   }
+  return codeTypes;
+}
 
-  if (jsonOutput) {
-    print(json.encode(jsonData));
-  } else {
-    print('==================================');
-    print(sb.toString().trim());
-    print('==================================');
-  }
+String getFilenameFor(int i) {
+  return "file${i.toString().padLeft(5, '0')}.dart";
 }
 
 enum CodeType {

pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_many_files_in_flutter_set_subscriptions.dart

@@ -4,7 +4,8 @@
 
 import '../language_server_benchmark.dart';
 import '../legacy_messages.dart';
-import 'utils.dart';
+import '../run_utils.dart';
+import 'benchmark_utils.dart';
 
 /// At least until "https://github.com/flutter/flutter-intellij/issues/7980" is
 /// fixed, when a user in IntelliJ in a Flutter project opens a file it's added
@@ -26,6 +27,8 @@ Future<void> main(List<String> args) async {
   await runHelper(
     args,
     LegacyManyFilesInFlutterSetSubscriptionsBenchmark.new,
+    copyData,
+    extraIterations: getExtraIterations,
     runAsLsp: false,
   );
 }

pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_many_get_fixes_and_get_assists_requests.dart

@@ -4,7 +4,8 @@
 
 import '../language_server_benchmark.dart';
 import '../legacy_messages.dart';
-import 'utils.dart';
+import '../run_utils.dart';
+import 'benchmark_utils.dart';
 
 /// In reports of the analyzer being slow we've seen `edit.getFixes` causing a
 /// long queue because they take longer to execute than the wait before the next
@@ -19,9 +20,11 @@ Future<void> main(List<String> args) async {
   await runHelper(
     args,
     LegacyManyGetFixesAndGetAssisstRequestsBenchmark.new,
+    copyData,
+    extraIterations: getExtraIterations,
     runAsLsp: false,
     // The number of files doesn't seem to be important on this one.
-    numberOfFileOptions: [4],
+    sizeOptions: [4],
   );
 }
 

pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_many_hover_requests.dart

@@ -4,7 +4,8 @@
 
 import '../language_server_benchmark.dart';
 import '../legacy_messages.dart';
-import 'utils.dart';
+import '../run_utils.dart';
+import 'benchmark_utils.dart';
 
 /// Hovering over an import with ctrl down in IntelliJ sends getHover requests
 /// for the import uri at position 0 every ~8 ms and never cancels old requests.
@@ -15,9 +16,11 @@ Future<void> main(List<String> args) async {
   await runHelper(
     args,
     LegacyManyHoverRequestsBenchmark.new,
+    copyData,
+    extraIterations: getExtraIterations,
     runAsLsp: false,
     // The number of files doesn't seem to be important on this one.
-    numberOfFileOptions: [4],
+    sizeOptions: [4],
   );
 }
 

pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_typing_temporary_missing_end_brace.dart

@@ -4,7 +4,8 @@
 
 import '../language_server_benchmark.dart';
 import '../legacy_messages.dart';
-import 'utils.dart';
+import '../run_utils.dart';
+import 'benchmark_utils.dart';
 
 /// In IntelliJ, typing `if (something) {` only automatically inserts the
 /// matching end brace `}` when hitting enter. This makes it "not unlikely"
@@ -20,6 +21,8 @@ Future<void> main(List<String> args) async {
   await runHelper(
     args,
     LegacyTypingTemporaryMissingEndBraceBenchmark.new,
+    copyData,
+    extraIterations: getExtraIterations,
     runAsLsp: false,
   );
 }

pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_with_plugin_that_times_out.dart

@@ -4,7 +4,8 @@
 
 import '../language_server_benchmark.dart';
 import '../legacy_messages.dart';
-import 'utils.dart';
+import '../run_utils.dart';
+import 'benchmark_utils.dart';
 
 /// We've observed that sometimes the plugin that users has installed times out
 /// (takes > 500 ms to answer). This benchmark simulates that and shows the
@@ -13,10 +14,12 @@ Future<void> main(List<String> args) async {
   await runHelper(
     args,
     LegacyWithPluginThatTimesOutBencmark.new,
+    copyData,
+    extraIterations: getExtraIterations,
     runAsLsp: false,
-    includePlugin: true,
+    extraInformation: (includePlugin: true),
     // The number of files isn't important here.
-    numberOfFileOptions: [10],
+    sizeOptions: [10],
   );
 }
 

pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/lsp_completion_after_change.dart

@@ -4,13 +4,20 @@
 
 import '../language_server_benchmark.dart';
 import '../lsp_messages.dart';
-import 'utils.dart';
+import '../run_utils.dart';
+import 'benchmark_utils.dart';
 
 /// Change a file in a big project and reports how long it takes before the
 /// analysis server is responsive again (measured by when it responds to a
 /// completion request) and when it's actually done analysing.
 Future<void> main(List<String> args) async {
-  await runHelper(args, LspCompletionAfterChange.new, runAsLsp: true);
+  await runHelper(
+    args,
+    LspCompletionAfterChange.new,
+    copyData,
+    extraIterations: getExtraIterations,
+    runAsLsp: true,
+  );
 }
 
 class LspCompletionAfterChange extends DartLanguageServerBenchmark {

pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/lsp_with_plugin_that_times_out.dart

@@ -4,7 +4,8 @@
 
 import '../language_server_benchmark.dart';
 import '../lsp_messages.dart';
-import 'utils.dart';
+import '../run_utils.dart';
+import 'benchmark_utils.dart';
 
 /// We've observed that sometimes the plugin that users has installed times out
 /// (takes > 500 ms to answer). This benchmark simulates that and shows the
@@ -13,10 +14,12 @@ Future<void> main(List<String> args) async {
   await runHelper(
     args,
     LspWithPluginThatTimesOutBencmark.new,
+    copyData,
+    extraIterations: getExtraIterations,
     runAsLsp: true,
-    includePlugin: true,
+    extraInformation: (includePlugin: true),
     // The number of files isn't important here.
-    numberOfFileOptions: [10],
+    sizeOptions: [10],
   );
 }