lint rules: rename and simplify the benchmarking script.

* Rename from `cli.dart` to `benchmark.dart`, as the purpose of this
  script is to benchmark the lint rules.
* Move some code from `analyzer_utils.dart` to `benchmark.dart`, and
  simplify, as these functions were only used in the benchmark script.
* Remove a number of command-line arguments that the script used to
  accept; some are now implied, and others are no longer used.


Change-Id: I8fce2375f093778b2bd9d9aaf21afe8495731b7e
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/388946
Reviewed-by: Phil Quitslund <[email protected]>
Commit-Queue: Samuel Rawlins <[email protected]>

Author: srawlins

pkg/linter/lib/src/test_utilities/analyzer_utils.dart

@@ -3,110 +3,3 @@
 // BSD-style license that can be found in the LICENSE file.
 
 // ignore_for_file: implementation_imports
-
-import 'dart:convert';
-import 'dart:io';
-import 'dart:math' as math;
-
-import 'package:analyzer/error/error.dart';
-import 'package:analyzer/src/lint/io.dart' show errorSink;
-
-import '../analyzer.dart';
-import 'test_linter.dart';
-
-Future<Iterable<AnalysisErrorInfo>> lintFiles(
-    TestLinter linter, List<File> filesToLint) async {
-  // Setup an error watcher to track whether an error was logged to stderr so
-  // we can set the exit code accordingly.
-  var errorWatcher = ErrorWatchingSink(errorSink);
-  errorSink = errorWatcher;
-
-  var errors = await linter.lintFiles(filesToLint);
-  if (errorWatcher.encounteredError) {
-    exitCode = loggedAnalyzerErrorExitCode;
-  } else if (errors.isNotEmpty) {
-    exitCode = _maxSeverity(errors.toList(), linter.options.filter);
-  }
-
-  return errors;
-}
-
-Iterable<AnalysisError> _filtered(
-        List<AnalysisError> errors, LintFilter? filter) =>
-    (filter == null)
-        ? errors
-        : errors.where((AnalysisError e) => !filter.filter(e));
-
-int _maxSeverity(List<AnalysisErrorInfo> errors, LintFilter? filter) {
-  var max = 0;
-  for (var info in errors) {
-    _filtered(info.errors, filter).forEach((AnalysisError e) {
-      max = math.max(max, e.errorCode.errorSeverity.ordinal);
-    });
-  }
-  return max;
-}
-
-class ErrorWatchingSink implements IOSink {
-  bool encounteredError = false;
-
-  IOSink delegate;
-
-  ErrorWatchingSink(this.delegate);
-
-  @override
-  Future<void> get done => delegate.done;
-
-  @override
-  Encoding get encoding => delegate.encoding;
-
-  @override
-  set encoding(Encoding encoding) {
-    delegate.encoding = encoding;
-  }
-
-  @override
-  void add(List<int> data) {
-    delegate.add(data);
-  }
-
-  @override
-  void addError(Object error, [StackTrace? stackTrace]) {
-    encounteredError = true;
-    delegate.addError(error, stackTrace);
-  }
-
-  @override
-  Future<void> addStream(Stream<List<int>> stream) =>
-      delegate.addStream(stream);
-
-  @override
-  Future<void> close() => delegate.close();
-
-  @override
-  Future<void> flush() => delegate.flush();
-
-  @override
-  void write(Object? obj) {
-    delegate.write(obj);
-  }
-
-  @override
-  void writeAll(Iterable<Object?> objects, [String separator = '']) {
-    delegate.writeAll(objects, separator);
-  }
-
-  @override
-  void writeCharCode(int charCode) {
-    delegate.writeCharCode(charCode);
-  }
-
-  @override
-  void writeln([Object? obj = '']) {
-    // 'Exception while using a Visitor to visit ...' (
-    if (obj.toString().startsWith('Exception')) {
-      encounteredError = true;
-    }
-    delegate.writeln(obj);
-  }
-}

pkg/linter/lib/src/test_utilities/test_linter.dart

@@ -43,16 +43,16 @@ class TestLinter implements AnalysisErrorListener {
   }) : _resourceProvider =
             resourceProvider ?? file_system.PhysicalResourceProvider.INSTANCE;
 
-  Future<Iterable<AnalysisErrorInfo>> lintFiles(List<File> files) async {
+  Future<List<AnalysisErrorInfo>> lintFiles(List<File> files) async {
     var errors = <AnalysisErrorInfo>[];
     var lintDriver = LintDriver(options, _resourceProvider);
     errors.addAll(await lintDriver.analyze(files.where(isDartFile)));
     numSourcesAnalyzed = lintDriver.numSourcesAnalyzed;
-    files.where(isPubspecFile).forEach((path) {
+    files.where(isPubspecFile).forEach((file) {
       numSourcesAnalyzed++;
       var errorsForFile = lintPubspecSource(
-        contents: path.readAsStringSync(),
-        sourcePath: _resourceProvider.pathContext.normalize(path.absolute.path),
+        contents: file.readAsStringSync(),
+        sourcePath: _resourceProvider.pathContext.normalize(file.absolute.path),
       );
       errors.addAll(errorsForFile);
     });

pkg/linter/tool/cli.dart renamed to pkg/linter/tool/benchmark.dart

@@ -2,8 +2,10 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:async';
+import 'dart:convert';
 import 'dart:io';
-import 'dart:math';
+import 'dart:math' as math;
 
 import 'package:analyzer/error/error.dart';
 import 'package:analyzer/src/lint/config.dart';
@@ -14,57 +16,53 @@ import 'package:glob/glob.dart';
 import 'package:linter/src/analyzer.dart';
 import 'package:linter/src/extensions.dart';
 import 'package:linter/src/rules.dart';
-import 'package:linter/src/test_utilities/analyzer_utils.dart';
 import 'package:linter/src/test_utilities/formatter.dart';
 import 'package:linter/src/test_utilities/test_linter.dart';
 
 import 'lint_sets.dart';
 
-/// Starts linting from the command-line.
+/// Benchmarks lint rules.
 Future<void> main(List<String> args) async {
   await runLinter(args);
 }
 
 const unableToProcessExitCode = 64;
 
-String? getRoot(List<String> paths) =>
-    paths.length == 1 && Directory(paths.first).existsSync()
-        ? paths.first
-        : null;
+Future<Iterable<AnalysisErrorInfo>> lintFiles(
+    TestLinter linter, List<File> filesToLint) async {
+  // Setup an error watcher to track whether an error was logged to stderr so
+  // we can set the exit code accordingly.
+  var errorWatcher = _ErrorWatchingSink(errorSink);
+  errorSink = errorWatcher;
+  var errors = await linter.lintFiles(filesToLint);
+  if (errorWatcher.encounteredError) {
+    exitCode = loggedAnalyzerErrorExitCode;
+  } else if (errors.isNotEmpty) {
+    exitCode = _maxSeverity(errors, linter.options.filter);
+  }
+
+  return errors;
+}
 
 void printUsage(ArgParser parser, IOSink out, [String? error]) {
-  var message = 'Lints Dart source files and pubspecs.';
+  var message = 'Benchmark lint rules.';
   if (error != null) {
     message = error;
   }
 
   out.writeln('''$message
-Usage: linter <file>
+Usage: benchmark.dart <file>
 ${parser.usage}
-
-For more information, see https://github.com/dart-lang/linter
 ''');
 }
 
-// TODO(pq): consider using `dart analyze` where possible
-// see: https://github.com/dart-lang/linter/pull/2537
 Future<void> runLinter(List<String> args) async {
-  // Force the rule registry to be populated.
   registerLintRules();
 
   var parser = ArgParser();
   parser
     ..addFlag('help',
         abbr: 'h', negatable: false, help: 'Show usage information.')
-    ..addFlag('stats',
-        abbr: 's', negatable: false, help: 'Show lint statistics.')
-    ..addFlag('benchmark', negatable: false, help: 'Show lint benchmarks.')
-    ..addFlag('visit-transitive-closure',
-        help: 'Visit the transitive closure of imported/exported libraries.')
-    ..addFlag('quiet', abbr: 'q', help: "Don't show individual lint errors.")
-    ..addFlag('machine',
-        help: 'Print results in a format suitable for parsing.',
-        negatable: false)
     ..addOption('config', abbr: 'c', help: 'Use configuration from this file.')
     ..addOption('dart-sdk', help: 'Custom path to a Dart SDK.')
     ..addMultiOption('rules',
@@ -85,7 +83,8 @@ Future<void> runLinter(List<String> args) async {
     return;
   }
 
-  if (options.rest.isEmpty) {
+  var paths = options.rest;
+  if (paths.isEmpty) {
     printUsage(parser, errorSink,
         'Please provide at least one file or directory to lint.');
     exitCode = unableToProcessExitCode;
@@ -122,52 +121,16 @@ Future<void> runLinter(List<String> args) async {
     linterOptions.dartSdkPath = customSdk;
   }
 
-  var stats = options['stats'] as bool;
-  var benchmark = options['benchmark'] as bool;
-  if (stats || benchmark) {
-    linterOptions.enableTiming = true;
-  }
+  linterOptions.enableTiming = true;
 
   var filesToLint = [
-    for (var path in options.rest)
+    for (var path in paths)
       ...collectFiles(path)
           .map((file) => file.path.toAbsoluteNormalizedPath())
           .map(File.new),
   ];
 
-  if (benchmark) {
-    await writeBenchmarks(outSink, filesToLint, linterOptions);
-    return;
-  }
-
-  var linter = TestLinter(linterOptions);
-
-  try {
-    var timer = Stopwatch()..start();
-    var errors = await lintFiles(linter, filesToLint);
-    timer.stop();
-
-    var commonRoot = getRoot(options.rest);
-    var machine = options['machine'] ?? false;
-    var quiet = options['quiet'] ?? false;
-    ReportFormatter(
-      errors,
-      linterOptions.filter,
-      outSink,
-      elapsedMs: timer.elapsedMilliseconds,
-      fileCount: linter.numSourcesAnalyzed,
-      fileRoot: commonRoot,
-      showStatistics: stats,
-      machineOutput: machine as bool,
-      quiet: quiet as bool,
-    ).write();
-    // ignore: avoid_catches_without_on_clauses
-  } catch (err, stack) {
-    errorSink.writeln('''An error occurred while linting
-  Please report it at: github.com/dart-lang/linter/issues
-$err
-$stack''');
-  }
+  await writeBenchmarks(outSink, filesToLint, linterOptions);
 }
 
 Future<void> writeBenchmarks(
@@ -178,11 +141,7 @@ Future<void> writeBenchmarks(
     lintRuleTimers.timers.forEach((n, t) {
       var timing = t.elapsedMilliseconds;
       var previous = timings[n];
-      if (previous == null) {
-        timings[n] = timing;
-      } else {
-        timings[n] = min(previous, timing);
-      }
+      timings[n] = previous == null ? timing : math.min(previous, timing);
     });
   }
 
@@ -208,6 +167,73 @@ Future<void> writeBenchmarks(
   out.writeTimings(stats, 0);
 }
 
+Iterable<AnalysisError> _filtered(
+        Iterable<AnalysisError> errors, LintFilter? filter) =>
+    (filter == null)
+        ? errors
+        : errors.where((AnalysisError e) => !filter.filter(e));
+
+int _maxSeverity(List<AnalysisErrorInfo> infos, LintFilter? filter) {
+  var allErrors = _filtered(infos.expand((i) => i.errors), filter);
+  return allErrors.fold(
+      0, (value, e) => math.max(value, e.errorCode.errorSeverity.ordinal));
+}
+
+class _ErrorWatchingSink implements IOSink {
+  bool encounteredError = false;
+
+  final IOSink delegate;
+
+  _ErrorWatchingSink(this.delegate);
+
+  @override
+  Future<void> get done => delegate.done;
+
+  @override
+  Encoding get encoding => delegate.encoding;
+
+  @override
+  set encoding(Encoding encoding) => delegate.encoding = encoding;
+
+  @override
+  void add(List<int> data) => delegate.add(data);
+
+  @override
+  void addError(Object error, [StackTrace? stackTrace]) {
+    encounteredError = true;
+    delegate.addError(error, stackTrace);
+  }
+
+  @override
+  Future<void> addStream(Stream<List<int>> stream) =>
+      delegate.addStream(stream);
+
+  @override
+  Future<void> close() => delegate.close();
+
+  @override
+  Future<void> flush() => delegate.flush();
+
+  @override
+  void write(Object? obj) => delegate.write(obj);
+
+  @override
+  void writeAll(Iterable<Object?> objects, [String separator = '']) =>
+      delegate.writeAll(objects, separator);
+
+  @override
+  void writeCharCode(int charCode) => delegate.writeCharCode(charCode);
+
+  @override
+  void writeln([Object? obj = '']) {
+    // 'Exception while using a Visitor to visit ...' (
+    if (obj.toString().startsWith('Exception')) {
+      encounteredError = true;
+    }
+    delegate.writeln(obj);
+  }
+}
+
 class _FileGlobFilter extends LintFilter {
   final List<Glob> includes;
   final List<Glob> excludes;