Add benchmarks for when there is mostly no codegen. (#4091)

Author: davidmorgan

_benchmark/bin/_benchmark.dart

@@ -25,6 +25,14 @@ final commandRunner =
         'build-repo-path',
         help: 'Path to build repo to benchmark.',
       )
+      ..argParser.addMultiOption(
+        'dependency-override-path',
+        help:
+            'Set a dependency override in generated pubspec.yaml, for '
+            'benchmarking local changes to builders. Example: '
+            '--dependency-override=built_value_generator='
+            '/path/to/local/package',
+      )
       ..argParser.addOption(
         'generator',
         help: 'Generator to benchmark.',
@@ -48,8 +56,8 @@ final commandRunner =
         allowed: Shape.values.map((e) => e.name).toList(),
       )
       ..argParser.addFlag(
-        'use-experimental-resolver',
-        help: 'Whether to pass `--use-experimental-resolver` to build_runner.',
+        'mostly-no-codegen',
+        help: 'Whether to generate mostly source with no codegen.',
       );
 
 Future<void> main(List<String> arguments) async {

_benchmark/lib/benchmarks/built_value_generator_benchmark.dart

@@ -56,9 +56,20 @@ ${config.dependencyOverrides}
         libraryNumber,
         benchmarkSize: size,
       );
-      workspace.write(
-        'lib/$libraryName',
-        source: '''
+      if (config.config.mostlyNoCodegen && libraryNumber > 1) {
+        workspace.write(
+          'lib/$libraryName',
+          source: '''
+// ignore_for_file: unused_import
+${[for (final importName in importNames) "import '$importName';"].join('\n')}
+
+class Value {}
+''',
+        );
+      } else {
+        workspace.write(
+          'lib/$libraryName',
+          source: '''
 // ignore_for_file: unused_import
 import 'package:built_value/built_value.dart';
 
@@ -71,7 +82,8 @@ abstract class Value implements Built<Value, ValueBuilder> {
   factory Value(void Function(ValueBuilder) updates) = _\$Value;
 }
 ''',
-      );
+        );
+      }
     }
   }
 }

_benchmark/lib/benchmarks/freezed_generator_benchmark.dart

@@ -60,9 +60,20 @@ ${config.dependencyOverrides}
         libraryNumber,
         benchmarkSize: size,
       );
-      workspace.write(
-        'lib/$libraryName',
-        source: '''
+      if (config.config.mostlyNoCodegen && libraryNumber > 1) {
+        workspace.write(
+          'lib/$libraryName',
+          source: '''
+// ignore_for_file: unused_import
+${[for (final importName in importNames) "import '$importName';"].join('\n')}
+
+class Value {}
+''',
+        );
+      } else {
+        workspace.write(
+          'lib/$libraryName',
+          source: '''
 // ignore_for_file: unused_import
 import 'package:freezed_annotation/freezed_annotation.dart';
 
@@ -75,7 +86,8 @@ class Value with _\$Value {
   const factory Value() = _Value;
 }
 ''',
-      );
+        );
+      }
     }
   }
 }

_benchmark/lib/benchmarks/json_serializable_generator_benchmark.dart

@@ -57,9 +57,20 @@ ${config.dependencyOverrides}
         libraryNumber,
         benchmarkSize: size,
       );
-      workspace.write(
-        'lib/$libraryName',
-        source: '''
+      if (config.config.mostlyNoCodegen && libraryNumber > 1) {
+        workspace.write(
+          'lib/$libraryName',
+          source: '''
+// ignore_for_file: unused_import
+${[for (final importName in importNames) "import '$importName';"].join('\n')}
+
+class Value {}
+''',
+        );
+      } else {
+        workspace.write(
+          'lib/$libraryName',
+          source: '''
 // ignore_for_file: unused_import
 import 'package:json_annotation/json_annotation.dart';
 
@@ -75,7 +86,8 @@ class Value {
   Map<String, dynamic> toJson() => _\$ValueToJson(this);
 }
 ''',
-      );
+        );
+      }
     }
   }
 }

_benchmark/lib/benchmarks/mockito_generator_benchmark.dart

@@ -17,6 +17,11 @@ class MockitoGeneratorBenchmark implements Benchmark {
 
   @override
   void create(RunConfig config) {
+    if (config.config.mostlyNoCodegen) {
+      throw UnsupportedError(
+        'Mockito benchmark does not support --mostly-no-codegen.',
+      );
+    }
     final workspace = config.workspace;
     final size = config.size;
 

_benchmark/lib/config.dart

@@ -13,26 +13,32 @@ import 'workspace.dart';
 /// Benchmark tool config.
 class Config {
   final String? buildRepoPath;
+  final Map<String, String> dependencyOverridePaths;
   final Generator generator;
   final List<Shape> shapes;
   final Directory rootDirectory;
   final List<int> sizes;
-  final bool useExperimentalResolver;
   final bool allowFailures;
+  bool mostlyNoCodegen;
 
   Config({
     required this.allowFailures,
     required this.buildRepoPath,
+    required this.dependencyOverridePaths,
     required this.generator,
     required this.rootDirectory,
     required this.sizes,
     required this.shapes,
-    required this.useExperimentalResolver,
+    required this.mostlyNoCodegen,
   });
 
   factory Config.fromArgResults(ArgResults argResults) => Config(
     allowFailures: argResults['allow-failures'] as bool,
     buildRepoPath: argResults['build-repo-path'] as String?,
+    dependencyOverridePaths: {
+      for (var s in argResults['dependency-override-path'] as List<String>)
+        s.split('=')[0]: s.split('=')[1],
+    },
     generator: Generator.values.singleWhere(
       (e) => e.packageName == argResults['generator'],
     ),
@@ -45,7 +51,7 @@ class Config {
         argResults['shape'] == null
             ? Shape.values
             : [Shape.values.singleWhere((e) => e.name == argResults['shape'])],
-    useExperimentalResolver: argResults['use-experimental-resolver'] as bool,
+    mostlyNoCodegen: argResults['mostly-no-codegen'] as bool,
   );
 }
 
@@ -69,11 +75,11 @@ class RunConfig {
   });
 
   String get dependencyOverrides {
-    final buildRepoPath = config.buildRepoPath;
-    if (buildRepoPath == null) return '';
+    final overrides = StringBuffer();
 
-    return '''
-dependency_overrides:
+    final buildRepoPath = config.buildRepoPath;
+    if (buildRepoPath != null) {
+      overrides.write('''
   build:
     path: $buildRepoPath/build
   build_config:
@@ -90,6 +96,21 @@ dependency_overrides:
     path: $buildRepoPath/build_test
   build_web_compilers:
     path: $buildRepoPath/build_web_compilers
+''');
+    }
+
+    for (final entry in config.dependencyOverridePaths.entries) {
+      overrides.write('''
+  ${entry.key}:
+    path: ${entry.value}
+''');
+    }
+
+    return overrides.isEmpty
+        ? ''
+        : '''
+dependency_overrides:
+$overrides
 ''';
   }
 }

_benchmark/lib/workspace.dart

@@ -60,7 +60,6 @@ class Workspace {
       'build_runner',
       'build',
       '-d',
-      if (config.useExperimentalResolver) '--use-experimental-resolver',
     ], workingDirectory: directory.path);
     var exitCode = await process.exitCode;
     if (exitCode == 0) {
@@ -81,7 +80,6 @@ class Workspace {
       'build_runner',
       'build',
       '-d',
-      if (config.useExperimentalResolver) '--use-experimental-resolver',
     ], workingDirectory: directory.path);
     exitCode = await process.exitCode;
     if (exitCode == 0) {
@@ -101,7 +99,6 @@ class Workspace {
       'build_runner',
       'build',
       '-d',
-      if (config.useExperimentalResolver) '--use-experimental-resolver',
     ], workingDirectory: directory.path);
     exitCode = await process.exitCode;
     if (exitCode == 0) {