Support running benchmarks in AOT mode. (#1486)

Support running benchmarks in AOT mode.

Many of the optimizations I'm trying have a relatively small
improvement. That can make it hard to tell if they are actually an
improvement if there is enough variance in the runs that they are lost
in the noise.

AOT builds don't need warm-up time and seem to have generally lower
variance than JIT runs. Also, they are how the shipped formatter is
run in `dart format`. So this adds support to the two benchmark scripts
for running themselves in AOT snapshot mode.

I added support for this directly to the scripts instead of just
manually compiling and running them on the command line (which works
fine) because this way it's less error-prone. I don't have to remember
to re-compile the AOT snapshot and risk using an old build which is an
easy mistake to make when the only behavioral difference is (possibly)
performance.

Also, tweaked the profiler output a little.

Author: munificent

benchmark/directory.dart

@@ -4,24 +4,29 @@
 
 import 'dart:io';
 
+import 'package:args/args.dart';
+import 'package:collection/collection.dart';
 import 'package:dart_style/dart_style.dart';
 import 'package:dart_style/src/constants.dart';
 import 'package:dart_style/src/profile.dart';
+import 'package:dart_style/src/testing/benchmark.dart';
 
 /// Reads a given directory of Dart files and repeatedly formats their contents.
 ///
 /// This allows getting profile information on a large real-world codebase
 /// while also factoring out the file IO time from the process.
-void main(List<String> args) {
-  if (args.length != 1) {
-    stderr.writeln('Usage: dart run directory.dart <directory to format>');
-    exit(65);
-  }
+void main(List<String> arguments) async {
+  var directory = await _parseArguments(arguments);
 
-  var directory = args[0];
   print('Listing entries in $directory...');
   var entries = Directory(directory).listSync(recursive: true);
 
+  // Make sure the benchmark is deterministic. The order shouldn't really
+  // matter for performance, but since the JIT is warming up as it goes through
+  // the files, different orders could potentially affect how it chooses to
+  // optimize.
+  entries.sortBy((entry) => entry.path);
+
   print('Reading sources...');
   var sources = <String>[];
   for (var entry in entries) {
@@ -65,3 +70,44 @@ void _runFormatter(String source) {
     print(error.message());
   }
 }
+
+/// Parses the command line arguments and options.
+///
+/// Returns the path to the directory that should be formatted.
+Future<String> _parseArguments(List<String> arguments) async {
+  var argParser = ArgParser();
+  argParser.addFlag('help', negatable: false, help: 'Show usage information.');
+  argParser.addFlag('aot',
+      negatable: false,
+      help: 'Whether the benchmark should run in AOT mode versus JIT.');
+
+  var argResults = argParser.parse(arguments);
+  if (argResults['help'] as bool) {
+    _usage(argParser, exitCode: 0);
+  }
+
+  if (argResults.rest.length != 1) {
+    stderr.writeln('Missing directory path to format.');
+    stderr.writeln();
+    _usage(argParser, exitCode: 0);
+  }
+
+  if (argResults['aot'] as bool) {
+    await rerunAsAot([argResults.rest.single]);
+  }
+
+  return argResults.rest.single;
+}
+
+/// Prints usage information.
+///
+/// If [exitCode] is non-zero, prints to stderr.
+Never _usage(ArgParser argParser, {required int exitCode}) {
+  var stream = exitCode == 0 ? stdout : stderr;
+
+  stream.writeln('dart benchmark/directory.dart [--aot] <directory>');
+  stream.writeln('');
+  stream.writeln(argParser.usage);
+
+  exit(exitCode);
+}

benchmark/run.dart

@@ -38,6 +38,9 @@ const _baselinePath = 'benchmark/baseline.json';
 
 final _benchmarkDirectory = p.dirname(p.fromUri(Platform.script));
 
+/// Whether to run a number of trials before measuring to warm up the JIT.
+bool _runWarmupTrials = false;
+
 /// Whether to use the short or tall style formatter.
 bool _isShort = false;
 
@@ -68,7 +71,9 @@ Future<void> main(List<String> arguments) async {
     }
   }
 
-  await _warmUp();
+  if (_runWarmupTrials) {
+    await _warmUp();
+  }
 
   var results = <(Benchmark, List<double>)>[];
   for (var benchmark in benchmarks) {
@@ -175,10 +180,15 @@ double _runTrial(DartFormatter formatter, ParseStringResult parseResult,
 Future<List<Benchmark>> _parseArguments(List<String> arguments) async {
   var argParser = ArgParser();
   argParser.addFlag('help', negatable: false, help: 'Show usage information.');
+  argParser.addFlag('aot',
+      negatable: false,
+      help: 'Whether the benchmark should run in AOT mode versus JIT.');
   argParser.addFlag('short',
       abbr: 's',
       negatable: false,
       help: 'Whether the formatter should use short or tall style.');
+  argParser.addFlag('no-warmup',
+      negatable: false, help: 'Skip the JIT warmup runs.');
   argParser.addFlag('write-baseline',
       abbr: 'w',
       negatable: false,
@@ -189,6 +199,14 @@ Future<List<Benchmark>> _parseArguments(List<String> arguments) async {
     _usage(argParser, exitCode: 0);
   }
 
+  if (argResults['aot'] as bool) {
+    await rerunAsAot([
+      for (var argument in arguments)
+        if (argument != '--aot') argument,
+      '--no-warmup',
+    ]);
+  }
+
   var benchmarks = switch (argResults.rest) {
     // Find all the benchmarks.
     ['all'] => await Benchmark.findAll(),
@@ -205,6 +223,7 @@ Future<List<Benchmark>> _parseArguments(List<String> arguments) async {
     _ => _usage(argParser, exitCode: 64),
   };
 
+  _runWarmupTrials = !(argResults['no-warmup'] as bool);
   _isShort = argResults['short'] as bool;
   _writeBaseline = argResults['write-baseline'] as bool;
 
@@ -250,7 +269,7 @@ Never _usage(ArgParser argParser, {required int exitCode}) {
   var stream = exitCode == 0 ? stdout : stderr;
 
   stream.writeln('dart benchmark/run.dart [benchmark/case/<benchmark>.unit] '
-      '[--short] [--baseline=n]');
+      '[--aot] [--short] [--baseline=n]');
   stream.writeln('');
   stream.writeln(argParser.usage);
 

lib/src/profile.dart

@@ -58,20 +58,22 @@ class Profile {
   static void begin(String label) {
     if (!enabled) return;
 
+    // Indent to show nesting of profiled regions.
+    label = '${'  ' * _running.length}$label';
+
     _running.add((label, Timeline.now));
+
+    // Add the label eagerly to the map so that labels are printed in the
+    // order they were began.
+    _accumulatedTimes.putIfAbsent(label, () => 0);
   }
 
-  static void end(String label) {
+  static void end(String _) {
     if (!enabled) return;
 
-    var (startLabel, start) = _running.removeLast();
-
-    // Must push and pop in stack order.
-    assert(label == startLabel);
+    var (label, start) = _running.removeLast();
     var elapsed = Timeline.now - start;
-
-    _accumulatedTimes.update(label, (accumulated) => accumulated + elapsed,
-        ifAbsent: () => elapsed);
+    _accumulatedTimes.update(label, (accumulated) => accumulated + elapsed);
   }
 
   /// Discards all recorded profiling data.
@@ -105,8 +107,6 @@ class Profile {
     print(header);
 
     var labels = data.keys.toList();
-    labels.sort();
-
     var longestLabel =
         labels.fold(0, (length, label) => max(length, label.length));
 

lib/src/testing/benchmark.dart

@@ -81,3 +81,35 @@ class Benchmark {
       required this.shortOutput,
       required this.tallOutput});
 }
+
+/// Compiles the currently running script to an AOT snapshot and then executes
+/// it.
+///
+/// This function never returns. When the AOT snapshot ends, this exits the
+/// process.
+Future<Never> rerunAsAot(List<String> arguments) async {
+  var script = Platform.script.toFilePath();
+  var snapshotPath = p.join(
+      Directory.systemTemp.path, p.setExtension(p.basename(script), '.aot'));
+
+  print('Creating AOT snapshot for $script...');
+  var result = await Process.run(
+      'dart', ['compile', 'aot-snapshot', '-o', snapshotPath, script]);
+  stdout.write(result.stdout);
+  stderr.write(result.stderr);
+  if (result.exitCode != 0) {
+    stderr.writeln('Failed to create AOT snapshot.');
+    exit(result.exitCode);
+  }
+
+  print('Running AOT snapshot...');
+  var process =
+      await Process.start('dartaotruntime', [snapshotPath, ...arguments]);
+  await stdout.addStream(process.stdout);
+  await stderr.addStream(process.stderr);
+
+  var exitCode = await process.exitCode;
+
+  await File(snapshotPath).delete();
+  exit(exitCode);
+}

lib/src/testing/test_file.dart

@@ -15,9 +15,16 @@ final _unicodeEscapePattern = RegExp('[\x0a\x0c\x0d]');
 
 /// Get the absolute local file path to the dart_style package's root directory.
 Future<String> findPackageDirectory() async {
-  var libraryUri = await Isolate.resolvePackageUri(
-      Uri.parse('package:dart_style/src/testing/test_file.dart'));
-  return p.normalize(p.join(p.dirname(libraryUri!.toFilePath()), '../../..'));
+  var libraryPath = (await Isolate.resolvePackageUri(
+          Uri.parse('package:dart_style/src/testing/test_file.dart')))
+      ?.toFilePath();
+
+  // Fallback, if we can't resolve the package URI because we're running in an
+  // AOT snapshot, just assume we're running from the root directory of the
+  // package.
+  libraryPath ??= 'lib/src/testing/test_file.dart';
+
+  return p.normalize(p.join(p.dirname(libraryPath), '../../..'));
 }
 
 /// Get the absolute local file path to the package's "test" directory.