diff --git a/benchmark/builder_parsing_benchmark.dart b/benchmark/builder_parsing.dart similarity index 100% rename from benchmark/builder_parsing_benchmark.dart rename to benchmark/builder_parsing.dart diff --git a/benchmark/dart2js_output.dart b/benchmark/dart2js_output.dart new file mode 100644 index 000000000..b5dbf590b --- /dev/null +++ b/benchmark/dart2js_output.dart @@ -0,0 +1,275 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:args/command_runner.dart'; +import 'package:meta/meta.dart'; + +import 'dart2js_output/compile.dart'; +import 'dart2js_output/dart2js_normalize.dart'; +import 'dart2js_output/logging.dart'; +import 'dart2js_output/source.dart' as source; + +Future main(List args) async { + final runner = CommandRunner( + 'benchmark-dart2js-output', + 'Runs various dart2js output benchmarks and comparisons.' + ' Useful for debugging/validating how changes to over_react affect dart2js output.') + ..addCommand(CompareSizeCommand()) + ..addCommand(CompareCodeCommand()) + ..addCommand(GetCodeCommand()); + + await runner.run(args); +} + +final _originMasterDep = jsonEncode({ + 'git': { + 'url': Directory.current.uri.toString(), + 'ref': 'origin/HEAD', + } +}); +final _localPathDep = jsonEncode({ + 'path': Directory.current.path, +}); + +abstract class BaseCommand extends Command { + BaseCommand() { + argParser.addFlag('verbose', defaultsTo: true, negatable: true); + } + + @override + @mustCallSuper + void run() { + initLogging(verbose: argResults!['verbose'] as bool); + } +} + +abstract class CompareCommand extends BaseCommand { + CompareCommand() { + argParser.addOption( + 'head', + help: 'Head over_react dependency to compare to the base.' + ' Defaults to the enclosing local working copy of over_react.', + defaultsTo: _localPathDep, + ); + argParser.addOption( + 'base', + help: 'Base over_react dependency to compare against.' + ' Defaults to origin/master.', + defaultsTo: _originMasterDep, + ); + } + + dynamic get _baseDep => jsonDecode(argResults!['base'] as String); + + dynamic get _headDep => jsonDecode(argResults!['head'] as String); +} + +class CompareSizeCommand extends CompareCommand { + @override + String get description => + 'Compares the optimized, minified size of dart2js output for a benchmark React component between two over_react versions.'; + + @override + String get name => 'compare-size'; + + @override + Future run() async { + super.run(); + final baseSize = getComponentAndUsageSize(overReactDep: _baseDep); + final headSize = getComponentAndUsageSize(overReactDep: _headDep); + print('Base size: ${await baseSize} bytes'); + print('Head size: ${await headSize} bytes'); + print('(Head size) - (base size):' + ' ${(await headSize) - (await baseSize)} bytes'); + } +} + +class CompareCodeCommand extends CompareCommand { + @override + String get name => 'compare-code'; + + @override + String get description => + 'Compares the optimized, non-minified dart2js output for a benchmark React component between two over_react versions.' + '\nOutputs in a Git diff format.' + '\nCompiled code is normalized before comparison for better diffing.'; + + @override + Future run() async { + super.run(); + final diff = await compareCodeAcrossVersions( + source.componentBenchmark( + componentCount: 1, + propsCount: 5, + ), + overReactDep1: _baseDep, + overReactDep2: _headDep, + color: stdioType(stdout) == StdioType.terminal, + ); + if (diff.trim().isEmpty) { + print('(No difference in dart2js output between base and head)'); + } else { + print(diff); + } + } +} + +class GetCodeCommand extends BaseCommand { + @override + String get name => 'get-code'; + + @override + String get description => + 'Displays the optimized, non-minified dart2js output for a benchmark React component.' + '\nOutputs in a Git diff format, showing output changes when adding a component.' + '\nCompiled code is normalized before comparison for better diffing.'; + + GetCodeCommand() { + argParser.addOption( + 'dependency', + help: 'over_react dependency to compile with.' + ' Defaults to the enclosing local working copy of over_react.', + defaultsTo: _localPathDep, + ); + } + + dynamic get _dep => jsonDecode(argResults!['dependency'] as String); + + @override + Future run() async { + super.run(); + print(await getCompiledComponentCode( + overReactDep: _dep, + color: stdioType(stdout) == StdioType.terminal, + )); + } +} + +Future compareCodeAcrossVersions( + String code, { + required dynamic overReactDep1, + required dynamic overReactDep2, + bool color = false, +}) async { + final results1 = compileOverReactProgram( + webFilesByName: {'main.dart': code}, + overReactDep: overReactDep1, + minify: false, + ); + final results2 = compileOverReactProgram( + webFilesByName: {'main.dart': code}, + overReactDep: overReactDep2, + minify: false, + ); + + return gitDiffNoIndex( + createNormalizedDart2jsFile((await results1).getCompiledDart2jsFile()).path, + createNormalizedDart2jsFile((await results2).getCompiledDart2jsFile()).path, + color: color, + ); +} + +Future getCompiledComponentCode({ + dynamic overReactDep, + bool color = false, +}) async { + const baselineComponentCount = 2; + const propsCount = 3; + + final result = await compileOverReactProgram(webFilesByName: { + 'baseline.dart': source.componentBenchmark( + componentCount: baselineComponentCount, + propsCount: propsCount, + ), + 'additional.dart': source.componentBenchmark( + componentCount: baselineComponentCount + 1, + propsCount: propsCount, + ), + }, overReactDep: overReactDep, minify: false); + + final baselineCompiledFile = result.getCompiledDart2jsFile('baseline.dart'); + final additionalCompiledFile = + result.getCompiledDart2jsFile('additional.dart'); + + return gitDiffNoIndex( + createNormalizedDart2jsFile(baselineCompiledFile).path, + createNormalizedDart2jsFile(additionalCompiledFile).path, + color: color, + ); +} + +File createNormalizedDart2jsFile(File f) { + return File(f.path + '.normalized.js') + ..writeAsStringSync(normalizeDart2jsContents(f.readAsStringSync())); +} + +Future gitDiffNoIndex( + String file1, + String file2, { + int contextLines = 1, + bool color = false, +}) async { + final result = await Process.run('git', [ + 'diff', + '--no-index', + '-U$contextLines', + if (color) '--color', + file1, + file2, + ]); + + if (result.exitCode == 0 || result.exitCode == 1) { + return result.stdout as String; + } + + throw Exception( + 'Error diffing files. Exit code: ${result.exitCode} stderr: $stderr'); +} + +/// Gets the total size of a single test component, plus usage that sets all props, +/// and render that reads all props. +/// +/// Since it contains this extra usage and render code, it's mainly useful when +/// comparing across versions, and shouldn't by itself be used as a number that +/// represents "the cost of declaring a component"." +Future getComponentAndUsageSize({ + dynamic overReactDep, +}) async { + const baselineComponentCount = 100; + const propsCount = 5; + + final result = await compileOverReactProgram(webFilesByName: { + 'baseline.dart': source.componentBenchmark( + componentCount: baselineComponentCount, + propsCount: propsCount, + ), + 'additional.dart': source.componentBenchmark( + componentCount: baselineComponentCount + 1, + propsCount: propsCount, + ), + }, overReactDep: overReactDep); + + final baselineFileSize = + result.getCompiledDart2jsFile('baseline.dart').statSync().size; + final additionalFileSize = + result.getCompiledDart2jsFile('additional.dart').statSync().size; + validateFileSize(baselineFileSize); + validateFileSize(additionalFileSize); + + return additionalFileSize - baselineFileSize; +} + +void validateFileSize(int actualSizeInBytes) { + // Arbitrary minimum expected size to help ensure the test setup is correct. + // + // Value derived from the compiled size of the following Dart program: + // import 'package:over_react/over_react.dart'; + // void main() { Dom.div()(); } + const minimumExpectedSizeInBytes = 144339; + + if (actualSizeInBytes < minimumExpectedSizeInBytes) { + throw Exception('Expected compiled size to be larger,' + ' at least $minimumExpectedSizeInBytes bytes.' + ' Was: $actualSizeInBytes bytes.'); + } +} diff --git a/benchmark/dart2js_output/compile.dart b/benchmark/dart2js_output/compile.dart new file mode 100644 index 000000000..276e74dfd --- /dev/null +++ b/benchmark/dart2js_output/compile.dart @@ -0,0 +1,173 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:logging/logging.dart'; +import 'package:path/path.dart' as p; + +final overReactPackageRoot = Directory.current; + +Future compileOverReactProgram({ + required Map webFilesByName, + dynamic overReactDep, + bool minify = true, +}) async { + overReactDep ??= { + 'path': overReactPackageRoot.path, + }; + + final logger = Logger('compileOverReactProgram'); + + final tempPath = Directory.systemTemp.createTempSync().path; + final packagePath = p.join(tempPath, 'package'); + final outputPath = p.join(tempPath, 'build_output'); + final packageDir = Directory(packagePath)..createSync(); + + logger.fine('Creating temporary package: $packagePath'); + + // YAML is a superset of JSON :) + File(p.join(packagePath, 'pubspec.yaml')).writeAsStringSync(jsonEncode({ + 'name': '_over_react_benchmark_test', + 'version': '0.0.0', + 'environment': { + 'sdk': '>=2.19.0 <4.0.0', + }, + 'dependencies': {'over_react': overReactDep}, + 'dev_dependencies': { + 'build_runner': 'any', + 'build_web_compilers': 'any', + } + })); + // YAML is a superset of JSON :) + File(p.join(packagePath, 'build.yaml')).writeAsStringSync(jsonEncode({ + 'targets': { + r'$default': { + 'builders': { + 'build_web_compilers:entrypoint': { + 'release_options': { + 'dart2js_args': [ + '--csp', + '-O3', + if (minify) '--minify' else '--no-minify', + '--verbose', + '--dump-info=binary', + ] + } + }, + // Need this to prevent dumped info from being output. + // See: https://github.com/dart-lang/build/issues/1622 + 'build_web_compilers:dart2js_archive_extractor': { + 'release_options': {'filter_outputs': false} + } + } + } + } + })); + + final webPath = p.join(packagePath, 'web'); + webFilesByName.forEach((name, contentsTemplate) { + final filePath = p.join(packagePath, 'web', name); + final partFilename = + p.basenameWithoutExtension(name) + '.over_react.g.dart'; + final contents = contentsTemplate.replaceAll('{{PART_PATH}}', partFilename); + + if (!p.isWithin(webPath, filePath)) { + throw ArgumentError.value( + name, 'Filename must be a relative path without any `..`.'); + } + File(filePath) + ..parent.createSync(recursive: true) + ..writeAsStringSync(contents); + }); + + const dartExecutable = 'dart'; + + logger.fine('Running pub get...'); + final pubGetResult = await Process.run( + dartExecutable, + ['pub', 'get'], + workingDirectory: packagePath, + runInShell: true, + ); + if (pubGetResult.exitCode != 0) { + throw Exception( + 'Unexpected `pub get` failure in temporary package: $packagePath\n' + '${pubGetResult.infoForErrorMessage}'); + } + + logger.fine('Running build...'); + + final buildResult = await Process.run( + dartExecutable, + [ + 'pub', + 'run', + 'build_runner', + 'build', + // Make sure to build in dart2js + '--release', + '--output', + outputPath, + 'web' + ], + workingDirectory: packagePath, + runInShell: true, + ); + logger.fine('Build complete; output to: $outputPath'); + File(p.join(outputPath, 'build_output.log')) + ..parent.createSync(recursive: true) + ..writeAsStringSync(buildResult.stdout.toString()); + if (buildResult.exitCode != 0) { + throw Exception( + 'Unexpected build failure in temporary package: $packagePath\n' + '${buildResult.infoForErrorMessage}'); + } + + await packageDir.delete(recursive: true); + logger.fine('Deleted temporary package.'); + + return BuildResult( + buildFolderPath: outputPath, + ); +} + +extension on ProcessResult { + String get infoForErrorMessage => 'Exit code: ${this.exitCode}.' + '\nstdout:\n${this.stdout}' + '\nstderr:\n${this.stderr}'; +} + +class BuildResult { + final String buildFolderPath; + + BuildResult({ + required this.buildFolderPath, + }); +} + +extension BuildResultUtils on BuildResult { + File getCompiledDart2jsFile([String? dartFilename]) { + final webFolder = Directory(p.join(buildFolderPath, 'web')); + + final File compiledFile; + if (dartFilename != null) { + compiledFile = File(p.join(webFolder.path, dartFilename + '.js')); + if (!compiledFile.existsSync()) { + throw Exception('Compiled file ${compiledFile.path} does not exist'); + } + } else { + final candidates = webFolder + .listSync() + .whereType() + .where((f) => f.path.endsWith('.dart.js')) + .toList(); + if (candidates.length != 1) { + throw Exception( + 'Expected a single dart2js output, but found ${candidates.length}:' + '${candidates.map((c) => '\n- ${c.path}')}'); + } + compiledFile = candidates.single; + } + + return compiledFile; + } +} diff --git a/benchmark/dart2js_output/dart2js_normalize.dart b/benchmark/dart2js_output/dart2js_normalize.dart new file mode 100644 index 000000000..c3c875845 --- /dev/null +++ b/benchmark/dart2js_output/dart2js_normalize.dart @@ -0,0 +1,58 @@ +import 'dart:collection'; +import 'dart:convert'; + +import 'package:collection/collection.dart'; + +/// Normalizes compiled dart2js code so it can be diffed cleanly against other +/// compiled dart2js code. +/// +/// Useful for isolating changes introduced by a program +/// +/// This logic is a bit brittle, but the goal is to improve readability/diffing, +/// and works well enough without having to pull in a JS formatter or something. +String normalizeDart2jsContents(String contents) => contents + // Replace numbers and identifiers that change when compiling additional code + .replaceAll(RegExp(r'\b\d+\b'), '###') // number literals + .replaceAll(RegExp(r'\b_static(_\d+)?\b'), 'static###') + .replaceAll(RegExp(r'\bt\d+\b'), 't#') // local variables (e.g., `t1`) + .replaceAllMapped(RegExp(r'\b(B\.\w+_)[0-9a-zA-Z]+\b'), (match) { + return match.group(1)! + '####'; // compile-time constant declarations + }) + // Remove newlines in empty function bodies + .replaceAllMapped( + RegExp(r'(function [\w$]+\(\) {)\s+}'), (match) => '${match[1]}}') + // Break up long type inheritance lists onto separate lines + .replaceAllMapped(RegExp(r'(_inheritMany\()([^\n]+)(\]\);)'), (match) { + return '${match[1]}${match[2]!.replaceAll(', ', ',\n ')},\n ${match[3]}'; + }) + // Break up long type metadata list onto separate lines, and sort + .replaceAllMapped(RegExp(r'types: (\["[^\n]+~[^\n]+"\]),'), (match) { + try { + final parsed = jsonDecode(match.group(1)!) as List; + return 'types: ${JsonEncoder.withIndent(' ').convert(deepSorted(parsed))},'; + } catch (_) {} + return match.group(0)!; + }) + // Try to format inlined JSON strings that contain type metadata + .replaceAllMapped(RegExp(r"JSON\.parse\('(\{[^\n]+\})'\)"), (match) { + final stringContents = match.group(1)!; + String formatted; + try { + formatted = const JsonEncoder.withIndent(' ') + .convert(deepSorted(jsonDecode(stringContents))); + } catch (_) { + formatted = stringContents.replaceAll(',"', ',\n"'); + } + return "#JSON_PARSE#'$formatted#/JSON_PARSE#"; + }); + +/// Returns a deep copy of [original], with all nested Lists and Maps sorted. +Object? deepSorted(Object? original) { + if (original is List) return original.map(deepSorted).toList()..sort(); + if (original is Map) { + return LinkedHashMap.fromEntries(original.entries + .sortedBy((entry) => entry.key.toString()) + .map((e) => MapEntry(e.key, deepSorted(e.value)))); + } + return original; +} diff --git a/benchmark/dart2js_output/logging.dart b/benchmark/dart2js_output/logging.dart new file mode 100644 index 000000000..ff72756c9 --- /dev/null +++ b/benchmark/dart2js_output/logging.dart @@ -0,0 +1,33 @@ +import 'dart:io'; + +import 'package:io/ansi.dart' as ansi; +import 'package:logging/logging.dart'; + +void initLogging({bool verbose = true}) { + Logger.root.level = verbose ? Level.ALL : Level.INFO; + Logger.root.onRecord.listen((rec) { + String? Function(String) colorizer; + IOSink output; + + if (rec.level >= Level.SEVERE) { + colorizer = ansi.red.wrap; + output = stderr; + } else if (rec.level >= Level.WARNING) { + colorizer = ansi.yellow.wrap; + output = stderr; + } else { + colorizer = (string) => string; + output = stdout; + } + + if (rec.message != '') { + output.writeln(colorizer('[${rec.level}] ${rec.message}')); + } + if (rec.error != null) { + output.writeln(colorizer(rec.error.toString())); + } + if (verbose && rec.stackTrace != null) { + output.writeln(colorizer(rec.stackTrace.toString())); + } + }); +} diff --git a/benchmark/dart2js_output/source.dart b/benchmark/dart2js_output/source.dart new file mode 100644 index 000000000..f0be91105 --- /dev/null +++ b/benchmark/dart2js_output/source.dart @@ -0,0 +1,67 @@ +String componentBenchmark({ + required int componentCount, + required int propsCount, +}) { + final fileSource = StringBuffer(''' + import 'package:over_react/over_react.dart'; + part '{{PART_PATH}}';'''); + + final mainStatements = StringBuffer()..writeln(mainAntiTreeShakingStatements); + for (var i = 0; i < componentCount; i++) { + final componentName = 'Foo$i'; + final mixinName = '${componentName}Props'; + final propsName = mixinName; + final propTypesByName = { + for (final i in Iterable.generate(propsCount)) 'foo$i': 'String', + }; + final propNames = propTypesByName.keys; + + fileSource.writeln(''' + mixin $mixinName on UiProps { + ${propDeclarations(propTypesByName)} + } + UiFactory<$propsName> $componentName = uiFunction((props) { + ${propReadStatements(propNames)} + final consumedProps = props.staticMeta.forMixins({$mixinName}); + return (Dom.div() + ..addUnconsumedProps(props, consumedProps) + ..modifyProps(props.addPropsToForward()) + )(); + }, _\$${componentName}Config); + '''); + + mainStatements.writeln('($componentName()${ + // Write each prop + propNames.map((name) => '..$name = ""').join('')})();'); + } + + fileSource + ..writeln('void main() {') + ..write(mainStatements) + ..writeln('}'); + + return fileSource.toString(); +} + +const mainAntiTreeShakingStatements = ''' + (Dom.div()..id = '1')(); // Other props class, DomProps + ResizeSensor()(); // class component, legacy component, PropsMeta used in propTypes + ResizeSensor().getPropKey((p) => p.id); // getPropKey generated impls +'''; + +String propDeclarations(Map propTypesByName) { + return propTypesByName + .mapEntries((name, type) => 'late $type $name;') + .join('\n'); +} + +String propReadStatements(Iterable propNames) { + // Put inside an array, as opposed to separate print statements, so that each + // additional prop doesn't contain additional code outside of the read. + return 'print([${propNames.map((name) => 'props.$name').join(', ')}]);'; +} + +extension on Map { + Iterable mapEntries(T Function(K, V) mapper) => + entries.map((entry) => mapper(entry.key, entry.value)); +} diff --git a/pubspec.yaml b/pubspec.yaml index e8a0deb9b..a1e91d4b8 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -27,6 +27,7 @@ dependencies: redux_dev_tools: '>=0.6.0 <0.8.0' dev_dependencies: + args: ^2.4.2 benchmark_harness: ^2.2.1 build_resolvers: ^2.0.0 build_runner: ^2.0.0