From eb1737b0331409db09e9af19863e8e03539978da Mon Sep 17 00:00:00 2001 From: Kevin Moore Date: Tue, 25 Nov 2025 12:58:56 -0800 Subject: [PATCH 1/5] [benchmark_harness] require Dart 3.10, format, prepare release --- .github/workflows/benchmark_harness.yaml | 2 +- pkgs/benchmark_harness/CHANGELOG.md | 3 +- pkgs/benchmark_harness/analysis_options.yaml | 2 + .../lib/src/async_benchmark_base.dart | 4 +- .../lib/src/bench_command/bench_options.dart | 61 +++++++++++-------- .../src/bench_command/compile_and_run.dart | 27 +++++--- .../lib/src/measurement.dart | 8 ++- .../lib/src/perf_benchmark_base.dart | 52 ++++++++++------ .../lib/src/score_emitter.dart | 16 +++-- pkgs/benchmark_harness/pubspec.yaml | 4 +- .../test/bench_command_test.dart | 47 +++++++------- .../test/result_emitter_test.dart | 2 +- 12 files changed, 138 insertions(+), 90 deletions(-) diff --git a/.github/workflows/benchmark_harness.yaml b/.github/workflows/benchmark_harness.yaml index 4e3de6e4f..07320d8e1 100644 --- a/.github/workflows/benchmark_harness.yaml +++ b/.github/workflows/benchmark_harness.yaml @@ -55,7 +55,7 @@ jobs: matrix: # Add macos-latest and/or windows-latest if relevant for this package. os: [ubuntu-latest] - sdk: [3.2, dev] + sdk: [3.10, dev] steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - uses: dart-lang/setup-dart@e51d8e571e22473a2ddebf0ef8a2123f0ab2c02c diff --git a/pkgs/benchmark_harness/CHANGELOG.md b/pkgs/benchmark_harness/CHANGELOG.md index 87864a084..f9afe69ae 100644 --- a/pkgs/benchmark_harness/CHANGELOG.md +++ b/pkgs/benchmark_harness/CHANGELOG.md @@ -1,6 +1,7 @@ -## 2.4.0-wip +## 2.4.0 - Added a `bench` command. +- Require `sdk: ^3.10.0`. ## 2.3.1 diff --git a/pkgs/benchmark_harness/analysis_options.yaml b/pkgs/benchmark_harness/analysis_options.yaml index 9a77232bb..f2a0e6990 100644 --- a/pkgs/benchmark_harness/analysis_options.yaml +++ b/pkgs/benchmark_harness/analysis_options.yaml @@ -1,3 +1,5 @@ +# https://dart.dev/guides/language/analysis-options + include: package:dart_flutter_team_lints/analysis_options.yaml analyzer: diff --git a/pkgs/benchmark_harness/lib/src/async_benchmark_base.dart b/pkgs/benchmark_harness/lib/src/async_benchmark_base.dart index 1472ee7a4..bd362021f 100644 --- a/pkgs/benchmark_harness/lib/src/async_benchmark_base.dart +++ b/pkgs/benchmark_harness/lib/src/async_benchmark_base.dart @@ -36,7 +36,9 @@ class AsyncBenchmarkBase { /// Measures the score for this benchmark by executing it repeatedly until /// time minimum has been reached. static Future measureFor( - Future Function() f, int minimumMillis) async { + Future Function() f, + int minimumMillis, + ) async { final minimumMicros = minimumMillis * 1000; final watch = Stopwatch()..start(); var iter = 0; diff --git a/pkgs/benchmark_harness/lib/src/bench_command/bench_options.dart b/pkgs/benchmark_harness/lib/src/bench_command/bench_options.dart index e1ffa1cdb..5ce10849d 100644 --- a/pkgs/benchmark_harness/lib/src/bench_command/bench_options.dart +++ b/pkgs/benchmark_harness/lib/src/bench_command/bench_options.dart @@ -7,7 +7,8 @@ import 'package:args/args.dart'; enum RuntimeFlavor { aot(help: 'Compile and run as a native binary.'), jit( - help: 'Run as-is without compilation, ' + help: + 'Run as-is without compilation, ' 'using the just-in-time (JIT) runtime.', ), js(help: 'Compile to JavaScript and run on node.'), @@ -36,13 +37,17 @@ class BenchOptions { final result = _parserForBenchOptions.parse(args); if (result.rest.isNotEmpty) { - throw FormatException('All arguments must be provided via `--` options. ' - 'Not sure what to do with "${result.rest.join()}".'); + throw FormatException( + 'All arguments must be provided via `--` options. ' + 'Not sure what to do with "${result.rest.join()}".', + ); } return BenchOptions( - flavor: - result.multiOption('flavor').map(RuntimeFlavor.values.byName).toSet(), + flavor: result + .multiOption('flavor') + .map(RuntimeFlavor.values.byName) + .toSet(), target: result.option('target')!, help: result.flag('help'), verbose: result.flag('verbose'), @@ -60,23 +65,31 @@ class BenchOptions { static String get usage => _parserForBenchOptions.usage; static final _parserForBenchOptions = ArgParser() - ..addMultiOption('flavor', - abbr: 'f', - allowed: RuntimeFlavor.values.map((e) => e.name), - allowedHelp: { - for (final flavor in RuntimeFlavor.values) flavor.name: flavor.help - }) - ..addOption('target', - defaultsTo: 'benchmark/benchmark.dart', - help: 'The target script to compile and run.') - ..addFlag('help', - defaultsTo: false, - negatable: false, - help: 'Print usage information and quit.', - abbr: 'h') - ..addFlag('verbose', - defaultsTo: false, - negatable: false, - help: 'Print the full stack trace if an exception is thrown.', - abbr: 'v'); + ..addMultiOption( + 'flavor', + abbr: 'f', + allowed: RuntimeFlavor.values.map((e) => e.name), + allowedHelp: { + for (final flavor in RuntimeFlavor.values) flavor.name: flavor.help, + }, + ) + ..addOption( + 'target', + defaultsTo: 'benchmark/benchmark.dart', + help: 'The target script to compile and run.', + ) + ..addFlag( + 'help', + defaultsTo: false, + negatable: false, + help: 'Print usage information and quit.', + abbr: 'h', + ) + ..addFlag( + 'verbose', + defaultsTo: false, + negatable: false, + help: 'Print the full stack trace if an exception is thrown.', + abbr: 'v', + ); } diff --git a/pkgs/benchmark_harness/lib/src/bench_command/compile_and_run.dart b/pkgs/benchmark_harness/lib/src/bench_command/compile_and_run.dart index bd8f29207..c1036f1bf 100644 --- a/pkgs/benchmark_harness/lib/src/bench_command/compile_and_run.dart +++ b/pkgs/benchmark_harness/lib/src/bench_command/compile_and_run.dart @@ -39,7 +39,7 @@ enum _Stage { compile, run } /// Base class for runtime-specific runners. abstract class _Runner { _Runner._({required this.target, required this.flavor}) - : assert(FileSystemEntity.isFileSync(target), '$target is not a file'); + : assert(FileSystemEntity.isFileSync(target), '$target is not a file'); factory _Runner({required RuntimeFlavor flavor, required String target}) { return (switch (flavor) { @@ -58,8 +58,9 @@ abstract class _Runner { /// /// Takes care of creating and deleting the corresponding temp directory. Future run() async { - _tempDirectory = Directory.systemTemp - .createTempSync('bench_${DateTime.now().millisecondsSinceEpoch}_'); + _tempDirectory = Directory.systemTemp.createTempSync( + 'bench_${DateTime.now().millisecondsSinceEpoch}_', + ); try { await _runImpl(); } finally { @@ -75,14 +76,20 @@ abstract class _Runner { /// Also prints out a nice message before execution denoting the [flavor] and /// the [stage]. Future _runProc( - _Stage stage, String executable, List args) async { + _Stage stage, + String executable, + List args, + ) async { print(''' \n${flavor.name.toUpperCase()} - ${stage.name.toUpperCase()} $executable ${args.join(' ')} '''); - final proc = await Process.start(executable, args, - mode: ProcessStartMode.inheritStdio); + final proc = await Process.start( + executable, + args, + mode: ProcessStartMode.inheritStdio, + ); final exitCode = await proc.exitCode; @@ -156,14 +163,16 @@ class _WasmRunner extends _Runner { outFile, ]); - final jsFile = - File.fromUri(_tempDirectory.uri.resolve('$_outputFileRoot.js')); + final jsFile = File.fromUri( + _tempDirectory.uri.resolve('$_outputFileRoot.js'), + ); jsFile.writeAsStringSync(_wasmInvokeScript); await _runProc(_Stage.run, 'node', [jsFile.path]); } - static const _wasmInvokeScript = ''' + static const _wasmInvokeScript = + ''' import { readFile } from 'node:fs/promises'; // For async file reading import { fileURLToPath } from 'url'; import { dirname, join } from 'path'; diff --git a/pkgs/benchmark_harness/lib/src/measurement.dart b/pkgs/benchmark_harness/lib/src/measurement.dart index 11119b983..b31147a99 100644 --- a/pkgs/benchmark_harness/lib/src/measurement.dart +++ b/pkgs/benchmark_harness/lib/src/measurement.dart @@ -12,8 +12,9 @@ Measurement measureForImpl(void Function() f, int minimumMillis) { final minimumMicros = minimumMillis * 1000; // If running a long measurement permit some amount of measurement jitter // to avoid discarding results that are almost good, but not quite there. - final allowedJitter = - minimumMillis < 1000 ? 0 : (minimumMicros * 0.1).floor(); + final allowedJitter = minimumMillis < 1000 + ? 0 + : (minimumMicros * 0.1).floor(); var iter = 2; var totalIterations = iter; final watch = Stopwatch()..start(); @@ -29,7 +30,8 @@ Measurement measureForImpl(void Function() f, int minimumMillis) { } iter = measurement.estimateIterationsNeededToReach( - minimumMicros: minimumMicros); + minimumMicros: minimumMicros, + ); totalIterations += iter; } } diff --git a/pkgs/benchmark_harness/lib/src/perf_benchmark_base.dart b/pkgs/benchmark_harness/lib/src/perf_benchmark_base.dart index a1c3de9c9..46f1b4f6b 100644 --- a/pkgs/benchmark_harness/lib/src/perf_benchmark_base.dart +++ b/pkgs/benchmark_harness/lib/src/perf_benchmark_base.dart @@ -19,8 +19,10 @@ class PerfBenchmarkBase extends BenchmarkBase { late final Process perfProcess; late final List perfProcessArgs; - PerfBenchmarkBase(super.name, - {ScoreEmitterV2 super.emitter = const PrintEmitterV2()}); + PerfBenchmarkBase( + super.name, { + ScoreEmitterV2 super.emitter = const PrintEmitterV2(), + }); ScoreEmitterV2 get _emitterV2 => emitter as ScoreEmitterV2; @@ -30,8 +32,12 @@ class PerfBenchmarkBase extends BenchmarkBase { for (final path in [perfControlFifo, perfControlAck]) { final fifoResult = await Process.run('mkfifo', [path]); if (fifoResult.exitCode != 0) { - throw ProcessException('mkfifo', [path], - 'Cannot create fifo: ${fifoResult.stderr}', fifoResult.exitCode); + throw ProcessException( + 'mkfifo', + [path], + 'Cannot create fifo: ${fifoResult.stderr}', + fifoResult.exitCode, + ); } } } @@ -70,27 +76,33 @@ class PerfBenchmarkBase extends BenchmarkBase { // Exit code from perf is -SIGINT when terminated with SIGINT. if (exitCode != 0 && exitCode != -ProcessSignal.sigint.signalNumber) { throw ProcessException( - 'perf', perfProcessArgs, lines.join('\n'), exitCode); + 'perf', + perfProcessArgs, + lines.join('\n'), + exitCode, + ); } - const metrics = { - 'cycles': 'CpuCycles', - 'page-faults': 'MajorPageFaults', - }; + const metrics = {'cycles': 'CpuCycles', 'page-faults': 'MajorPageFaults'}; for (final line in lines) { - if (line.split('\t') - case [ - String counter, - _, - String event && ('cycles' || 'page-faults'), - ... - ]) { - _emitterV2.emit(name, double.parse(counter) / totalIterations, - metric: metrics[event]!); + if (line.split('\t') case [ + String counter, + _, + String event && ('cycles' || 'page-faults'), + ..., + ]) { + _emitterV2.emit( + name, + double.parse(counter) / totalIterations, + metric: metrics[event]!, + ); } } - _emitterV2.emit('$name.totalIterations', totalIterations.toDouble(), - metric: 'Count'); + _emitterV2.emit( + '$name.totalIterations', + totalIterations.toDouble(), + metric: 'Count', + ); } /// Measures the score for the benchmark and returns it. diff --git a/pkgs/benchmark_harness/lib/src/score_emitter.dart b/pkgs/benchmark_harness/lib/src/score_emitter.dart index 440711861..1bdb9a72a 100644 --- a/pkgs/benchmark_harness/lib/src/score_emitter.dart +++ b/pkgs/benchmark_harness/lib/src/score_emitter.dart @@ -20,8 +20,12 @@ class PrintEmitter implements ScoreEmitter { /// be deprecated and removed. That release will be a breaking change. abstract class ScoreEmitterV2 implements ScoreEmitter { @override - void emit(String testName, double value, - {String metric = 'RunTime', String unit}); + void emit( + String testName, + double value, { + String metric = 'RunTime', + String unit, + }); } /// New implementation of [PrintEmitter] implementing the [ScoreEmitterV2] @@ -31,8 +35,12 @@ class PrintEmitterV2 implements ScoreEmitterV2 { const PrintEmitterV2(); @override - void emit(String testName, double value, - {String metric = 'RunTime', String unit = ''}) { + void emit( + String testName, + double value, { + String metric = 'RunTime', + String unit = '', + }) { print(['$testName($metric):', value, if (unit.isNotEmpty) unit].join(' ')); } } diff --git a/pkgs/benchmark_harness/pubspec.yaml b/pkgs/benchmark_harness/pubspec.yaml index b2ad9a045..a23b14b76 100644 --- a/pkgs/benchmark_harness/pubspec.yaml +++ b/pkgs/benchmark_harness/pubspec.yaml @@ -1,5 +1,5 @@ name: benchmark_harness -version: 2.4.0-wip +version: 2.4.0 description: The official Dart project benchmark harness. repository: https://github.com/dart-lang/tools/tree/main/pkgs/benchmark_harness issue_tracker: https://github.com/dart-lang/tools/issues?q=is%3Aissue+is%3Aopen+label%3Apackage%3Abenchmark_harness @@ -8,7 +8,7 @@ topics: - benchmarking environment: - sdk: ^3.2.0 + sdk: ^3.10.0 dependencies: args: ^2.5.0 diff --git a/pkgs/benchmark_harness/test/bench_command_test.dart b/pkgs/benchmark_harness/test/bench_command_test.dart index 77604160c..8cedb950f 100644 --- a/pkgs/benchmark_harness/test/bench_command_test.dart +++ b/pkgs/benchmark_harness/test/bench_command_test.dart @@ -14,22 +14,21 @@ import 'package:test/test.dart'; void main() { test('readme', () async { - final output = - await Process.run(Platform.executable, ['bin/bench.dart', '--help']); + final output = await Process.run(Platform.executable, [ + 'bin/bench.dart', + '--help', + ]); expect(output.exitCode, 0); // Sadly, the help output likes to include trailing spaces that don't // copy paste nicely or consistently into the README. - final trimmed = LineSplitter.split(output.stdout as String) - .map((e) => e.trimRight()) - .join('\n'); + final trimmed = LineSplitter.split( + output.stdout as String, + ).map((e) => e.trimRight()).join('\n'); final readmeFile = File('README.md').readAsStringSync(); - expect( - readmeFile, - contains(trimmed), - ); + expect(readmeFile, contains(trimmed)); }); group('invoke the command', () { @@ -50,9 +49,12 @@ void main() { group('BenchOptions.fromArgs', () { test('options parsing', () async { - final options = BenchOptions.fromArgs( - ['--flavor', 'aot,jit', '--target', testFilePath], - ); + final options = BenchOptions.fromArgs([ + '--flavor', + 'aot,jit', + '--target', + testFilePath, + ]); await expectLater( () => compileAndRun(options), @@ -70,9 +72,7 @@ void main() { test('rest args not supported', () async { expect( - () => BenchOptions.fromArgs( - ['--flavor', 'aot,jit', testFilePath], - ), + () => BenchOptions.fromArgs(['--flavor', 'aot,jit', testFilePath]), throwsFormatException, ); }); @@ -82,17 +82,16 @@ void main() { test('$bench', () async { await expectLater( () => compileAndRun( - BenchOptions(flavor: {bench}, target: testFilePath)), + BenchOptions(flavor: {bench}, target: testFilePath), + ), prints( - stringContainsInOrder( - [ - if (bench != RuntimeFlavor.jit) ...[ - '${bench.name.toUpperCase()} - COMPILE', - testFilePath, - ], - '${bench.name.toUpperCase()} - RUN' + stringContainsInOrder([ + if (bench != RuntimeFlavor.jit) ...[ + '${bench.name.toUpperCase()} - COMPILE', + testFilePath, ], - ), + '${bench.name.toUpperCase()} - RUN', + ]), ), ); }, skip: _skipWasm(bench)); diff --git a/pkgs/benchmark_harness/test/result_emitter_test.dart b/pkgs/benchmark_harness/test/result_emitter_test.dart index e2cd1eaa9..196b536df 100644 --- a/pkgs/benchmark_harness/test/result_emitter_test.dart +++ b/pkgs/benchmark_harness/test/result_emitter_test.dart @@ -21,7 +21,7 @@ class MockResultEmitter extends ScoreEmitter { // Create a new benchmark which has an emitter. class BenchmarkWithResultEmitter extends BenchmarkBase { const BenchmarkWithResultEmitter(ScoreEmitter emitter) - : super('Template', emitter: emitter); + : super('Template', emitter: emitter); @override void run() {} From d97d6e43e0b07d136dc212969c239e605c7c0f98 Mon Sep 17 00:00:00 2001 From: Kevin Moore Date: Tue, 25 Nov 2025 13:01:31 -0800 Subject: [PATCH 2/5] tiny tweaks to readme --- pkgs/benchmark_harness/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkgs/benchmark_harness/README.md b/pkgs/benchmark_harness/README.md index d4778087f..4513edf37 100644 --- a/pkgs/benchmark_harness/README.md +++ b/pkgs/benchmark_harness/README.md @@ -42,26 +42,26 @@ In other words, don't compare apples with oranges. ## Getting Started -1\. Add the following to your project's **pubspec.yaml** +1. Add the following to your project's **pubspec.yaml** ```yaml dependencies: benchmark_harness: any ``` -2\. Install pub packages +2. Install pub packages ```sh dart pub install ``` -3\. Add the following import: +3. Add the following import: ```dart import 'package:benchmark_harness/benchmark_harness.dart'; ``` -4\. Create a benchmark class which inherits from `BenchmarkBase` or +4. Create a benchmark class which inherits from `BenchmarkBase` or `AsyncBenchmarkBase`. ## Example @@ -108,7 +108,7 @@ void main() { ### Output ```console -Template(RunTime): 0.1568472448997197 us. +Template(RunTime): 0.1568472448997197 µs. ``` This is the average amount of time it takes to run `run()` 10 times for From 30712b7e9b2411818a954ae5cd6549152866f448 Mon Sep 17 00:00:00 2001 From: Kevin Moore Date: Tue, 25 Nov 2025 13:07:13 -0800 Subject: [PATCH 3/5] tiny tweak --- pkgs/benchmark_harness/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/benchmark_harness/README.md b/pkgs/benchmark_harness/README.md index 4513edf37..8ad873a3c 100644 --- a/pkgs/benchmark_harness/README.md +++ b/pkgs/benchmark_harness/README.md @@ -108,12 +108,12 @@ void main() { ### Output ```console -Template(RunTime): 0.1568472448997197 µs. +Template(RunTime): 0.1568472448997197 us. ``` This is the average amount of time it takes to run `run()` 10 times for `BenchmarkBase` and once for `AsyncBenchmarkBase`. -> µs is an abbreviation for microseconds. +> `us` is an abbreviation for microseconds. ## `bench` command From 4342b604cefe8751aeb4d2399f6fc1dac4063450 Mon Sep 17 00:00:00 2001 From: Kevin Moore Date: Tue, 25 Nov 2025 13:11:05 -0800 Subject: [PATCH 4/5] fix version silly Due to https://github.com/dart-lang/setup-dart/issues/161 --- .github/workflows/benchmark_harness.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark_harness.yaml b/.github/workflows/benchmark_harness.yaml index 07320d8e1..a0d4cdaa8 100644 --- a/.github/workflows/benchmark_harness.yaml +++ b/.github/workflows/benchmark_harness.yaml @@ -55,7 +55,9 @@ jobs: matrix: # Add macos-latest and/or windows-latest if relevant for this package. os: [ubuntu-latest] - sdk: [3.10, dev] + # Have to put `3.10.0` instead of `3.10` because of + # https://github.com/actions/setup-dart/issues/116 + sdk: [3.10.0, dev] steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - uses: dart-lang/setup-dart@e51d8e571e22473a2ddebf0ef8a2123f0ab2c02c From 8757571599f8694e928fddfe730e58d5aa135435 Mon Sep 17 00:00:00 2001 From: Kevin Moore Date: Tue, 25 Nov 2025 13:54:32 -0800 Subject: [PATCH 5/5] Apply suggestions from code review --- .github/workflows/benchmark_harness.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark_harness.yaml b/.github/workflows/benchmark_harness.yaml index a0d4cdaa8..ed35ce2f1 100644 --- a/.github/workflows/benchmark_harness.yaml +++ b/.github/workflows/benchmark_harness.yaml @@ -55,9 +55,9 @@ jobs: matrix: # Add macos-latest and/or windows-latest if relevant for this package. os: [ubuntu-latest] - # Have to put `3.10.0` instead of `3.10` because of - # https://github.com/actions/setup-dart/issues/116 - sdk: [3.10.0, dev] + # Have to put `'3.10'` instead of `3.10` because of + # https://github.com/dart-lang/setup-dart/issues/161 + sdk: ['3.10', dev] steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - uses: dart-lang/setup-dart@e51d8e571e22473a2ddebf0ef8a2123f0ab2c02c