[front_end] Fix bootstrapping of message generation.

The dart formatter depends on `package:analyzer`, which depends on the
code generated diagnostic message file
`pkg/_fe_analyzer_shared/lib/src/messages/codes_generated.dart`. This
means that if the code generator for this file tries to format it
using the formatter's programmatic API (the `DartFormatter` class),
there will be a bootstrapping problem, because the code generator will
include the generated file in its transitive imports.

To avoid this, `generate_messages.dart` is changed so that it first
writes out an unformatted file, and then uses the `DartFormat` class
(from `package:analyzer_utilities`) to do the formatting. The
`DartFormat` class runs `dart format` as a subprocess, so it doesn't
depend have to import `package:analyzer`.

In addition to fixing the bootstrapping problem, this has two side
benefits:

- By avoiding the import of `package:analyzer`, it makes the code
  generator run much faster. I see approximately a 4x speedup.

- Previously, if formatting failed due to a syntax error in the
  generated code, it no output would be generated, making the bug
  difficult to diagnose. Now, if the generated code contains a syntax
  error, the unformatted file is left on disk.

A previous fix for the bootstrapping problem
`generate_messages_failsafe.dart`, didn't have these advantages. It is
removed since it's no longer needed.

The test `generated_files_up_to_date_git_test.dart` is updated so that
it does the formatting using the `DartFormatter` class. This isn't a
problem for bootstrapping, because this test is only required to work
properly when the code is in a fully generated state.

Note that the code generator can also be invoked indirectly through
the `pkg/front_end/tool/cfe.dart` tool. This, unfortunately, still has
the bootstrapping problem, so a hint message is added to `cfe.dart` to
encourage the user to run `generate_messages.dart` directly.

Change-Id: I6a6a6964fec4259a4462c6ec5204fea2dcd28314
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/447920
Commit-Queue: Paul Berry <[email protected]>
Reviewed-by: Johnni Winther <[email protected]>

Author: stereotype441

pkg/analyzer/tool/messages/error_code_info.dart

@@ -476,7 +476,7 @@ class CfeToAnalyzerErrorCodeTables {
         throw '''
 $frontEndCode specifies index $index but indices must be 1 or greater.
 For more information run:
-pkg/front_end/tool/fasta generate-messages
+dart pkg/front_end/tool/generate_messages.dart
 ''';
       }
       if (indexToInfo.length <= index) {

pkg/analyzer/tool/messages/generate.dart

@@ -12,7 +12,7 @@
 /// containing the error and the name of the error separated by a `.`
 /// (e.g. ParserErrorCode.EQUALITY_CANNOT_BE_EQUALITY_OPERAND).
 ///
-/// It is expected that 'pkg/front_end/tool/fasta generate-messages'
+/// It is expected that 'dart pkg/front_end/tool/generate_messages.dart'
 /// has already been successfully run.
 library;
 

pkg/front_end/messages.yaml

@@ -4,14 +4,14 @@
 
 # Run
 #
-#     dart pkg/front_end/tool/cfe.dart generate-messages
+#     dart pkg/front_end/tool/generate_messages.dart
 #
 # to regenerate messages after having edited this file.
 #
 # If entries with 'analyzerCode', 'sharedName', or 'index' have been changed,
 # run both
 #
-#     dart pkg/front_end/tool/cfe.dart generate-messages
+#     dart pkg/front_end/tool/generate_messages.dart
 #     dart pkg/analyzer/tool/messages/generate.dart
 #
 # to regenerate analyzer messages.

pkg/front_end/pubspec.yaml

@@ -23,6 +23,7 @@ dependencies:
 # Use 'any' constraints here; we get our versions from the DEPS file.
 dev_dependencies:
   analyzer: any
+  analyzer_utilities: any
   args: any
   build_integration: any
   compiler: any

pkg/front_end/test/generated_files_up_to_date_git_test.dart

@@ -4,6 +4,8 @@
 
 import "dart:io" show File, exitCode;
 
+import 'package:dart_style/dart_style.dart' show DartFormatter;
+
 import "../tool/generate_experimental_flags.dart" as generateExperimentalFlags;
 import "../tool/generate_messages.dart" as generateMessages;
 import "../tool/parser_ast_helper_creator.dart" as generateParserAstHelper;
@@ -12,7 +14,7 @@ import '../tool/generate_ast_coverage.dart' as generateAstCoverage;
 import '../tool/generate_ast_equivalence.dart' as generateAstEquivalence;
 import "parser_test_listener_creator.dart" as generateParserTestListener;
 import "parser_test_parser_creator.dart" as generateParserTestParser;
-import 'utils/io_utils.dart' show computeRepoDirUri;
+import 'utils/io_utils.dart' show computeRepoDirUri, getPackageVersionFor;
 
 final Uri repoDir = computeRepoDirUri();
 
@@ -131,16 +133,20 @@ void messages() {
 
   Uri generatedFile = generateMessages.computeSharedGeneratedFile(repoDir);
   check(
-    messages.sharedMessages,
+    new DartFormatter(
+      languageVersion: getPackageVersionFor('_fe_analyzer_shared'),
+    ).format(messages.sharedMessages),
     generatedFile,
-    "dart pkg/front_end/tool/cfe.dart generate-messages",
+    "dart pkg/front_end/tool/generate_messages.dart",
   );
 
   Uri cfeGeneratedFile = generateMessages.computeCfeGeneratedFile(repoDir);
   check(
-    messages.cfeMessages,
+    new DartFormatter(
+      languageVersion: getPackageVersionFor('front_end'),
+    ).format(messages.cfeMessages),
     cfeGeneratedFile,
-    "dart pkg/front_end/tool/cfe.dart generate-messages",
+    "dart pkg/front_end/tool/generate_messages.dart",
   );
 }
 

pkg/front_end/test/spell_checking_list_tests.txt

@@ -107,6 +107,7 @@ bold
 boo
 boollist
 bools
+bootstrapping
 bots
 boundaries
 boundarykey

pkg/front_end/tool/cfe.dart

@@ -96,6 +96,10 @@ Future<void> main(List<String> args) async {
       scriptArguments.add('--config=${repoDir}/pkg/front_end/testing.json');
       break;
     case 'generate-messages':
+      print(
+        'Hint: running `generate_messages.dart` directly is faster and avoids '
+        'bootstrapping problems.',
+      );
       mainFunction = generate_messages.main;
       script = '${toolDir}/generate_messages.dart';
       break;

pkg/front_end/tool/generate_messages.dart

@@ -4,7 +4,7 @@
 
 import 'dart:io' show File;
 
-import 'package:dart_style/dart_style.dart' show DartFormatter;
+import 'package:analyzer_utilities/tools.dart';
 
 import '../test/utils/io_utils.dart' show computeRepoDirUri;
 import 'generate_messages_lib.dart';
@@ -21,18 +21,22 @@ void main(List<String> arguments) {
       "Refusing to overwrite with empty file!",
     );
   } else {
-    new File.fromUri(
-      computeSharedGeneratedFile(repoDir),
-    ).writeAsStringSync(message.sharedMessages, flush: true);
-    new File.fromUri(
-      computeCfeGeneratedFile(repoDir),
-    ).writeAsStringSync(message.cfeMessages, flush: true);
+    _writeAndFormat(
+      new File.fromUri(computeSharedGeneratedFile(repoDir)),
+      message.sharedMessages,
+    );
+    _writeAndFormat(
+      new File.fromUri(computeCfeGeneratedFile(repoDir)),
+      message.cfeMessages,
+    );
   }
 }
 
+void _writeAndFormat(File file, String contents) {
+  file.writeAsStringSync(contents, flush: true);
+  DartFormat.formatFile(file);
+}
+
 Messages generateMessagesFiles(Uri repoDir) {
-  return generateMessagesFilesRaw(
-    repoDir,
-    (s, version) => new DartFormatter(languageVersion: version).format(s),
-  );
+  return generateMessagesFilesRaw(repoDir);
 }

pkg/front_end/tool/generate_messages_failsafe.dart

@@ -6,11 +6,8 @@ import 'dart:io' show File, exitCode;
 
 import "package:_fe_analyzer_shared/src/messages/severity.dart"
     show severityEnumNames;
-import 'package:pub_semver/pub_semver.dart' show Version;
 import 'package:yaml/yaml.dart' show loadYaml;
 
-import '../test/utils/io_utils.dart';
-
 Uri computeSharedGeneratedFile(Uri repoDir) {
   return repoDir.resolve(
     "pkg/_fe_analyzer_shared/lib/src/messages/codes_generated.dart",
@@ -30,10 +27,7 @@ class Messages {
   Messages(this.sharedMessages, this.cfeMessages);
 }
 
-Messages generateMessagesFilesRaw(
-  Uri repoDir,
-  String Function(String, Version) formatter,
-) {
+Messages generateMessagesFilesRaw(Uri repoDir) {
   Uri messagesFile = repoDir.resolve("pkg/front_end/messages.yaml");
   Map<dynamic, dynamic> yaml = loadYaml(
     new File.fromUri(messagesFile).readAsStringSync(),
@@ -151,10 +145,7 @@ part of 'cfe_codes.dart';
     return new Messages('', '');
   }
 
-  return new Messages(
-    formatter("$sharedMessages", getPackageVersionFor("_fe_analyzer_shared")),
-    formatter("$cfeMessages", getPackageVersionFor("front_end")),
-  );
+  return new Messages("$sharedMessages", "$cfeMessages");
 }
 
 final RegExp placeholderPattern = new RegExp(