[kernel/CFE] Dill extractor tool; CFE compile dir tool; fixes for stats

This CL adds two tools
* A dill extractor tool which extracts the source files inside the dill
  to a new directory, trying to recrease the package config too so the
  sources can be compiled again. Thought as being useful for doing
  benchmarks between the CFE and the analyzer to make sure they look at
  the exact same source files. No more. No less.
* A CFE compile entry point that compiles all files in a directory.
  Again thought as being useful to compare the CFE with the Analyzer.

Additionally it introduces "--gcs=<int>" to `benchmarker.dart` so it can
do multiple runs and do statistics on combined gc times.

Usage example:

```
$ out/ReleaseX64/dart pkg/front_end/tool/compile.dart pkg/front_end/tool/compile.dart

$ out/ReleaseX64/dart-sdk/bin/dart pkg/kernel/bin/dill_extractor.dart pkg/front_end/tool/compile.dart.dill /tmp/extracted_compile_dart_compile
Done. Wrote 687 source files.

$ out/ReleaseX64/dart-sdk/bin/dart compile aot-snapshot pkg/front_end/tool/compile_files_in_folders.dart
Generated: [...]/pkg/front_end/tool/compile_files_in_folders.aot

$ out/ReleaseX64/dart-sdk/bin/dart compile aot-snapshot pkg/analyzer_cli/bin/analyzer.dart
Generated: [...]/pkg/analyzer_cli/bin/analyzer.aot

$ time out/ReleaseX64/dart-sdk/bin/dartaotruntime pkg/front_end/tool/compile_files_in_folders.aot /tmp/extracted_compile_dart_compile/
Got 626 libraries.
Finished in 0:00:03.496817

real    0m3.530s
user    0m4.985s
sys     0m0.257s

$ time out/ReleaseX64/dart-sdk/bin/dartaotruntime pkg/analyzer_cli/bin/analyzer.aot /tmp/extracted_compile_dart_compile/
Analyzing /tmp/extracted_compile_dart_compile...
  warning • Target of URI doesn't exist: 'package:compiler/src/io/source_file.dart'. • package:_fe_analyzer_shared/src/scanner/utf8_bytes_scanner.dart:5:16 • uri_does_not_exist_in_doc_import
1 warning found.

real    0m8.479s
user    0m9.997s
sys     0m1.048s


$ out/ReleaseX64/dart pkg/front_end/tool/benchmarker.dart --silent --iterations=10 --gcs=5 --snapshot=pkg/front_end/tool/compile_files_in_folders.aot --snapshot=pkg/analyzer_cli/bin/analyzer.aot --arguments="/tmp/extracted_compile_dart_compile/"
Will now run 10 iterations with 2 snapshots.
..............................

Comparing snapshot #1 (compile_files_in_folders.aot) with snapshot #2 (analyzer.aot)
msec task-clock:u: 141.1012% +/- 1.5550% (6281.55 +/- 69.22) (4451.81 -> 10733.36)
page-faults:u: 47.5414% +/- 0.7786% (45602.90 +/- 746.84) (95922.50 -> 141525.40)
cycles:u: 119.3182% +/- 1.5751% (22131655652.90 +/- 292155654.08) (18548436164.50 -> 40680091817.40)
instructions:u: 140.2463% +/- 0.0340% (30410953910.60 +/- 7375789.67) (21683959961.20 -> 52094913871.80)
branch-misses:u: 97.9816% +/- 7.3049% (73071988.00 +/- 5447773.30) (74577266.40 -> 147649254.40)
seconds time elapsed: 141.0789% +/- 1.5479% (6.28 +/- 0.07) (4.45 -> 10.74)
seconds user: 124.4165% +/- 1.8322% (5.31 +/- 0.08) (4.26 -> 9.57)
seconds sys: 521.0422% +/- 17.4291% (0.98 +/- 0.03) (0.19 -> 1.16)
Comparing GC:
Combined GC time: 63.8901% +/- 1.2946% (1008.76 +/- 20.44) (1578.90 -> 2587.66)
```

(note that benchmarker passes `--deterministic` and limits the run to one cpu, both of which (potentially) makes it slower --- which is why the stats say 4.45 s instead of ~3.53 s and 10.74 s instead of ~8.48 s)

Change-Id: I785620ce40af11ca1f7b6c88a0ac863b2200f7d3
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/449180
Reviewed-by: Johnni Winther <[email protected]>
Commit-Queue: Jens Johansen <[email protected]>

Author: jensjoha

pkg/front_end/test/simple_stats.dart

@@ -5,9 +5,9 @@
 import 'dart:math' as math;
 
 class SimpleTTestStat {
-  static TTestResult ttest<E extends num>(List<E> a, List<E> b) {
-    E aSum = a.reduce((value, element) => (value + element) as E);
-    E bSum = b.reduce((value, element) => (value + element) as E);
+  static TTestResult ttest(List<num> a, List<num> b) {
+    num aSum = sum(a);
+    num bSum = sum(b);
     int aCount = a.length;
     int bCount = b.length;
     double aMean = aSum / aCount;
@@ -41,18 +41,16 @@ class SimpleTTestStat {
     }
   }
 
-  static double average<E extends num>(List<E> data) {
-    E sum = data.reduce((value, element) => (value + element) as E);
-    return sum / data.length;
+  static double average(List<num> data) {
+    return sum(data) / data.length;
   }
 
-  static double variance<E extends num>(List<E> data) {
-    E sum = data.reduce((value, element) => (value + element) as E);
+  static double variance(List<num> data) {
     int count = data.length;
-    double average = sum / count;
+    double average = sum(data) / count;
 
     double diffSquareSum = 0;
-    for (E value in data) {
+    for (num value in data) {
       double diff = value - average;
       double squared = diff * diff;
       diffSquareSum += squared;
@@ -62,6 +60,14 @@ class SimpleTTestStat {
     return variance;
   }
 
+  static num sum(List<num> data) {
+    num result = 0;
+    for (num value in data) {
+      result += value;
+    }
+    return result;
+  }
+
   static double tTableTwoTails_0_05(int value) {
     // TODO: Maybe actually calculate this? I haven't looked that up.
     // These numbers are from google sheets "=TINV(0.05, value)"

pkg/front_end/test/spell_checking_list_code.txt

@@ -1324,6 +1324,7 @@ pieces
 pipe
 pkg
 pkgs
+placement
 play
 player
 plugin

pkg/front_end/test/spell_checking_list_tests.txt

@@ -388,6 +388,7 @@ gave
 gc
 gcd
 gclient
+gcs
 ge
 gesture
 gi

pkg/front_end/tool/benchmarker.dart

@@ -16,13 +16,16 @@ void main(List<String> args) {
   bool silent = false;
   int iterations = 5;
   int core = 7;
+  int gcRuns = 1;
   String? aotRuntime;
   String? checkFileSize;
   List<String> snapshots = [];
   List<String> arguments = [];
   for (String arg in args) {
     if (arg.startsWith("--iterations=")) {
       iterations = int.parse(arg.substring("--iterations=".length));
+    } else if (arg.startsWith("--gcs=")) {
+      gcRuns = int.parse(arg.substring("--gcs=".length));
     } else if (arg.startsWith("--core=")) {
       core = int.parse(arg.substring("--core=".length));
     } else if (arg.startsWith("--aotruntime")) {
@@ -59,6 +62,7 @@ void main(List<String> args) {
     checkFileSize,
     cacheBenchmarking: false,
     silent: silent,
+    gcRuns: gcRuns,
   );
   if (doCacheBenchmarkingToo) {
     _doRun(
@@ -70,6 +74,7 @@ void main(List<String> args) {
       checkFileSize,
       cacheBenchmarking: true,
       silent: silent,
+      gcRuns: gcRuns,
     );
   }
 }
@@ -83,16 +88,19 @@ void _doRun(
   String? checkFileSize, {
   required bool cacheBenchmarking,
   required bool silent,
+  required int gcRuns,
 }) {
   print(
     "Will now run $iterations iterations with "
     "${snapshots.length} snapshots.",
   );
 
   List<List<Map<String, num>>> runResults = [];
-  List<GCInfo> gcInfos = [];
+  List<List<GCInfo>> gcInfos = [];
   Warnings warnings = new Warnings();
   for (String snapshot in snapshots) {
+    List<GCInfo> gcInfo = [];
+    gcInfos.add(gcInfo);
     List<Map<String, num>> snapshotResults = [];
     runResults.add(snapshotResults);
     for (int iteration = 0; iteration < iterations; iteration++) {
@@ -118,11 +126,13 @@ void _doRun(
       snapshotResults.add(benchmarkRun);
     }
 
-    // Do a single GC run too.
-    if (silent) stdout.write(".");
-    gcInfos.add(
-      _verboseGcRun(aotRuntime, snapshot, [], arguments, silent: true),
-    );
+    // Do GC runs too.
+    for (int i = 0; i < gcRuns; i++) {
+      if (silent) stdout.write(".");
+      gcInfo.add(
+        _verboseGcRun(aotRuntime, snapshot, [], arguments, silent: true),
+      );
+    }
   }
   stdout.write("\n\n");
 
@@ -137,7 +147,20 @@ void _doRun(
     if (!_compare(firstSnapshotResults, compareToResults)) {
       print("No change.");
     }
-    printGcDiff(gcInfos.first, gcInfos[i]);
+    print("Comparing GC:");
+    if (gcRuns >= 3) {
+      if (!_compareSingle(
+        gcInfos[i].map((gcInfo) => gcInfo.combinedTime).toList(),
+        gcInfos[0].map((gcInfo) => gcInfo.combinedTime).toList(),
+        "Combined GC time",
+      )) {
+        print("No change in combined time.");
+      }
+    } else {
+      for (int gcNum = 0; gcNum < gcRuns; gcNum++) {
+        printGcDiff(gcInfos[0][gcNum], gcInfos[i][gcNum]);
+      }
+    }
   }
 
   if (warnings.scalingInEffect) {
@@ -213,19 +236,28 @@ bool _compare(List<Map<String, num>> from, List<Map<String, num>> to) {
       }
     }
     if (fromForCaption.isEmpty || toForCaption.isEmpty) continue;
-    TTestResult stats = SimpleTTestStat.ttest(toForCaption, fromForCaption);
-    if (stats.significant) {
-      somethingWasSignificant = true;
-      print(
-        "$caption: ${stats.percentChangeIfSignificant(fractionDigits: 4)} "
-        "(${stats.valueChangeIfSignificant(fractionDigits: 2)}) "
-        "(${stats.meanChangeStringIfSignificant(fractionDigits: 2)})",
-      );
-    }
+    somethingWasSignificant = _compareSingle(
+      toForCaption,
+      fromForCaption,
+      caption,
+    );
   }
   return somethingWasSignificant;
 }
 
+bool _compareSingle(List<num> to, List<num> from, String caption) {
+  TTestResult stats = SimpleTTestStat.ttest(to, from);
+  if (stats.significant) {
+    print(
+      "$caption: ${stats.percentChangeIfSignificant(fractionDigits: 4)} "
+      "(${stats.valueChangeIfSignificant(fractionDigits: 2)}) "
+      "(${stats.meanChangeStringIfSignificant(fractionDigits: 2)})",
+    );
+    return true;
+  }
+  return false;
+}
+
 List<num> _extractDataForCaption(String caption, List<Map<String, num>> data) {
   List<num> result = [];
   for (Map<String, num> entry in data) {
@@ -463,6 +495,10 @@ GCInfo parseVerboseGcText(List<String> stderrLines) {
   return new GCInfo(combinedTime, countWhat);
 }
 
+void combinedGcDiff(List<GCInfo> prev, List<GCInfo> current) {
+  prev.map((gcInfo) => gcInfo.combinedTime).toList();
+}
+
 void printGcDiff(GCInfo prev, GCInfo current) {
   Set<String> allKeys = {...prev.countWhat.keys, ...current.countWhat.keys};
   bool printedAnything = false;

pkg/front_end/tool/compile_files_in_folders.dart

@@ -0,0 +1,93 @@
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:_fe_analyzer_shared/src/messages/severity.dart';
+import 'package:front_end/src/api_prototype/compiler_options.dart' as api;
+import 'package:front_end/src/api_prototype/incremental_kernel_generator.dart'
+    show IncrementalCompilerResult;
+import 'package:front_end/src/base/processed_options.dart';
+import 'package:front_end/src/compute_platform_binaries_location.dart'
+    show computePlatformBinariesLocation;
+import 'package:front_end/src/base/compiler_context.dart';
+import 'package:front_end/src/base/incremental_compiler.dart';
+import 'package:kernel/target/targets.dart';
+import 'package:vm/modular/target/vm.dart';
+
+import '../test/utils/io_utils.dart' show computeRepoDirUri;
+
+final Uri repoDir = computeRepoDirUri();
+
+Future<void> main(List<String> args) async {
+  Stopwatch stopwatch = new Stopwatch()..start();
+  await run(args);
+  print("Finished in ${stopwatch.elapsed}");
+}
+
+Future<void> run(List<String> args) async {
+  api.CompilerOptions compilerOptions = getOptions();
+
+  ProcessedOptions options = new ProcessedOptions(options: compilerOptions);
+
+  Set<Uri> libUris = {};
+  Set<Uri> packageConfigUris = {};
+  for (String arg in args) {
+    Uri dir = Uri.base.resolveUri(new Uri.file(arg));
+    libUris.add(dir);
+    while (!File.fromUri(
+      dir.resolve(".dart_tool/package_config.json"),
+    ).existsSync()) {
+      Uri newDir = dir.resolve("..");
+      if (newDir != dir) {
+        dir = newDir;
+      } else {
+        throw "Couldn't find package config for $arg";
+      }
+    }
+    packageConfigUris.add(dir.resolve(".dart_tool/package_config.json"));
+  }
+  if (packageConfigUris.length != 1) throw "Didn't find unique package config.";
+
+  Uri packageConfigUri = packageConfigUris.first;
+  if (!new File.fromUri(packageConfigUri).existsSync()) {
+    throw "Couldn't find .dart_tool/package_config.json";
+  }
+  compilerOptions.packagesFileUri = packageConfigUri;
+
+  for (Uri uri in libUris) {
+    List<FileSystemEntity> entities = new Directory.fromUri(
+      uri,
+    ).listSync(recursive: true);
+    for (FileSystemEntity entity in entities) {
+      if (entity is File && entity.path.endsWith(".dart")) {
+        options.inputs.add(entity.uri);
+      }
+    }
+  }
+
+  IncrementalCompiler compiler = new IncrementalCompiler(
+    new CompilerContext(options),
+  );
+  IncrementalCompilerResult result = await compiler.computeDelta();
+  print("Got ${result.component.libraries.length} libraries.");
+}
+
+api.CompilerOptions getOptions() {
+  Uri sdkRoot = computePlatformBinariesLocation(forceBuildDir: true);
+  api.CompilerOptions options = new api.CompilerOptions()
+    ..sdkRoot = sdkRoot
+    ..compileSdk = false
+    ..target = new VmTarget(new TargetFlags())
+    ..librariesSpecificationUri = repoDir.resolve("sdk/lib/libraries.json")
+    ..omitPlatform = true
+    ..onDiagnostic = (api.CfeDiagnosticMessage message) {
+      if (message.severity == CfeSeverity.error) {
+        print(message.plainTextFormatted.join('\n'));
+        exitCode = 1;
+      }
+    }
+    ..environmentDefines = const {};
+  return options;
+}