[analyzer] Break dependency between analyzer message codegen and analyzer.

stereotype441 · Commit Queue · commit 6610e5cbfe43 · 2025-07-25T13:18:00.000-07:00
This fixes a bootstrapping problem: previously we could get into a situation where the generated files for analyzer diagnostic messages were out of date, but the code generator for those messages couldn't be run, because it imported the analyzer itself, which contained compiled errors due to the generated files being out of date. There were two problematic dependencies: - The `GeneratedContent.check` and `GeneratedContent.checkAll` methods relied on invoking the Dart formatter as a library (rather than invoking `dart format` as a subprocess); the Dart formatter library imports the analyzer. Fortunately, this functionality is not needed by the code generators themselves. It's only needed by the _tests_ that verify that code generation has run. So I was able to break the dependency by moving this functionality into a new file, `package:analyzer_utilities/generated_content_check.dart`, which is imported only by the code generation tests. The new API is slightly different: `GeneratedContent.checkAll` has been replaced by an extension method `check` on `Iterable<GeneratedContent>`. - The diagnostic message code generator used to have a post-processing step that made use of the scanner to find error messages used by the parser that needed to be translated, and issued a warning if it found any. This check was useful a long time ago, when we were unifying the analyzer and CFE parsers, and is no longer useful, so I've removed it. Change-Id: I509b741faca3b9ae21c23936001105ded731b490 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/442241 Reviewed-by: Konstantin Shcheglov <scheglov@google.com> Reviewed-by: Samuel Rawlins <srawlins@google.com> Commit-Queue: Paul Berry <paulberry@google.com>
diff --git a/pkg/analysis_server/tool/spec/check_all_test.dart b/pkg/analysis_server/tool/spec/check_all_test.dart
@@ -3,7 +3,7 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'package:analyzer_testing/package_root.dart' as pkg_root;
-import 'package:analyzer_utilities/tools.dart';
+import 'package:analyzer_utilities/generated_content_check.dart';
 import 'package:path/path.dart';
 
 import 'generate_all.dart';
@@ -12,9 +12,8 @@ import 'generate_all.dart';
 /// user to run generate_all.dart.
 Future<void> main() async {
   var pkgPath = normalize(join(pkg_root.packageRoot, 'analysis_server'));
-  await GeneratedContent.checkAll(
+  await allTargets.check(
     pkg_root.packageRoot,
     join(pkgPath, 'tool', 'spec', 'generate_all.dart'),
-    allTargets,
   );
 }
diff --git a/pkg/analysis_server_plugin/tool/api/generate_test.dart b/pkg/analysis_server_plugin/tool/api/generate_test.dart
@@ -6,16 +6,15 @@
 // date.
 
 import 'package:analyzer_testing/package_root.dart' as pkg_root;
-import 'package:analyzer_utilities/tools.dart';
+import 'package:analyzer_utilities/generated_content_check.dart';
 import 'package:path/path.dart';
 
 import 'generate.dart';
 
 Future<void> main() async {
-  await GeneratedContent.checkAll(
+  await allTargets.check(
     pkg_root.packageRoot,
     join(pkg_root.packageRoot, 'analysis_server_plugin', 'tool', 'api',
         'generate.dart'),
-    allTargets,
   );
 }
diff --git a/pkg/analyzer/tool/api/generate_test.dart b/pkg/analyzer/tool/api/generate_test.dart
@@ -6,15 +6,14 @@
 // date.
 
 import 'package:analyzer_testing/package_root.dart' as pkg_root;
-import 'package:analyzer_utilities/tools.dart';
+import 'package:analyzer_utilities/generated_content_check.dart';
 import 'package:path/path.dart';
 
 import 'generate.dart';
 
 Future<void> main() async {
-  await GeneratedContent.checkAll(
+  await allTargets.check(
     pkg_root.packageRoot,
     join(pkg_root.packageRoot, 'analyzer', 'tool', 'api', 'generate.dart'),
-    allTargets,
   );
 }
diff --git a/pkg/analyzer/tool/experiments/experiments_test.dart b/pkg/analyzer/tool/experiments/experiments_test.dart
@@ -3,7 +3,7 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'package:analyzer_testing/package_root.dart' as pkg_root;
-import 'package:analyzer_utilities/tools.dart';
+import 'package:analyzer_utilities/generated_content_check.dart';
 import 'package:path/path.dart';
 
 import 'generate.dart';
@@ -12,9 +12,8 @@ import 'generate.dart';
 /// user to run `generate.dart`.
 void main() async {
   String pkgPath = normalize(join(pkg_root.packageRoot, 'analyzer'));
-  await GeneratedContent.checkAll(
+  await allTargets.check(
     pkg_root.packageRoot,
     join(pkgPath, 'tool', 'experiments', 'generate.dart'),
-    allTargets,
   );
 }
diff --git a/pkg/analyzer/tool/messages/generate.dart b/pkg/analyzer/tool/messages/generate.dart
@@ -19,7 +19,6 @@ library;
 import 'dart:convert';
 import 'dart:io';
 
-import 'package:_fe_analyzer_shared/src/scanner/scanner.dart';
 import 'package:analyzer_testing/package_root.dart' as pkg_root;
 import 'package:analyzer_utilities/tools.dart';
 import 'package:collection/collection.dart';
@@ -384,52 +383,6 @@ class _SyntacticErrorGenerator {
         print('  ${analyzerToFasta.length} total');
       }
     }
-
-    // List error codes in the parser that have not been translated.
-    var untranslatedFastaErrorCodes = <String>{};
-    Token token = scanString(parserSource).tokens;
-    while (!token.isEof) {
-      if (token.isIdentifier) {
-        String? fastaErrorCode;
-        String lexeme = token.lexeme;
-        if (lexeme.length > 7) {
-          if (lexeme.startsWith('message')) {
-            fastaErrorCode = lexeme.substring(7);
-          } else if (lexeme.length > 8) {
-            if (lexeme.startsWith('template')) {
-              fastaErrorCode = lexeme.substring(8);
-            }
-          }
-        }
-        if (fastaErrorCode != null &&
-            cfeToAnalyzerErrorCodeTables.frontEndCodeToInfo[fastaErrorCode] ==
-                null) {
-          untranslatedFastaErrorCodes.add(fastaErrorCode);
-        }
-      }
-      token = token.next!;
-    }
-    if (untranslatedFastaErrorCodes.isNotEmpty) {
-      print('');
-      print('The following error codes in the parser are not auto generated:');
-      var sorted = untranslatedFastaErrorCodes.toList()..sort();
-      for (String fastaErrorCode in sorted) {
-        String analyzerCode = '';
-        String problemMessage = '';
-        var entry = frontEndMessages[fastaErrorCode];
-        if (entry != null) {
-          // TODO(paulberry): handle multiple analyzer codes
-          if (entry.index == null && entry.analyzerCode.length == 1) {
-            analyzerCode = entry.analyzerCode.single;
-            problemMessage = entry.problemMessage;
-          }
-        }
-        print(
-          '  ${fastaErrorCode.padRight(30)} --> $analyzerCode'
-          '\n      $problemMessage',
-        );
-      }
-    }
   }
 }
 
diff --git a/pkg/analyzer/tool/messages/generate_test.dart b/pkg/analyzer/tool/messages/generate_test.dart
@@ -6,16 +6,15 @@
 // date.
 
 import 'package:analyzer_testing/package_root.dart' as pkg_root;
-import 'package:analyzer_utilities/tools.dart';
+import 'package:analyzer_utilities/generated_content_check.dart';
 import 'package:path/path.dart';
 
 import 'error_code_info.dart';
 import 'generate.dart';
 
 Future<void> main() async {
-  await GeneratedContent.checkAll(
+  await allTargets.check(
     pkg_root.packageRoot,
     join(analyzerPkgPath, 'tool', 'messages', 'generate.dart'),
-    allTargets,
   );
 }
diff --git a/pkg/analyzer/tool/summary/check_test.dart b/pkg/analyzer/tool/summary/check_test.dart
@@ -3,7 +3,7 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'package:analyzer_testing/package_root.dart' as package_root;
-import 'package:analyzer_utilities/tools.dart';
+import 'package:analyzer_utilities/generated_content_check.dart';
 import 'package:path/path.dart';
 
 import 'generate.dart';
@@ -15,9 +15,7 @@ void main() async {
     join(package_root.packageRoot, 'analyzer', 'lib', 'src', 'summary'),
   );
   var idlPath = normalize(join(idlFolderPath, 'idl.dart'));
-  await GeneratedContent.checkAll(
-    package_root.packageRoot,
-    'pkg/analyzer/tool/summary/generate.dart',
-    getAllTargets(idlPath),
-  );
+  await getAllTargets(
+    idlPath,
+  ).check(package_root.packageRoot, 'pkg/analyzer/tool/summary/generate.dart');
 }
diff --git a/pkg/analyzer/tool/wolf/generate_test.dart b/pkg/analyzer/tool/wolf/generate_test.dart
@@ -6,15 +6,14 @@
 // date.
 
 import 'package:analyzer_testing/package_root.dart' as pkg_root;
-import 'package:analyzer_utilities/tools.dart';
+import 'package:analyzer_utilities/generated_content_check.dart';
 import 'package:path/path.dart';
 
 import 'generate.dart';
 
 Future<void> main() async {
-  await GeneratedContent.checkAll(
+  await allTargets.check(
     pkg_root.packageRoot,
     join(analyzerPkgPath, 'tool', 'wolf', 'generate.dart'),
-    allTargets,
   );
 }
diff --git a/pkg/analyzer_plugin/tool/api/generate_test.dart b/pkg/analyzer_plugin/tool/api/generate_test.dart
@@ -6,16 +6,15 @@
 // date.
 
 import 'package:analyzer_testing/package_root.dart' as pkg_root;
-import 'package:analyzer_utilities/tools.dart';
+import 'package:analyzer_utilities/generated_content_check.dart';
 import 'package:path/path.dart';
 
 import 'generate.dart';
 
 Future<void> main() async {
-  await GeneratedContent.checkAll(
+  await allTargets.check(
     pkg_root.packageRoot,
     join(pkg_root.packageRoot, 'analyzer_plugin', 'tool', 'api',
         'generate.dart'),
-    allTargets,
   );
 }
diff --git a/pkg/analyzer_plugin/tool/spec/check_all_test.dart b/pkg/analyzer_plugin/tool/spec/check_all_test.dart
@@ -3,7 +3,7 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'package:analyzer_testing/package_root.dart' as pkg_root;
-import 'package:analyzer_utilities/tools.dart';
+import 'package:analyzer_utilities/generated_content_check.dart';
 import 'package:path/path.dart';
 
 import 'generate_all.dart';
@@ -12,6 +12,8 @@ import 'generate_all.dart';
 /// user to run generate_all.dart.
 Future<void> main() async {
   var pkgPath = normalize(join(pkg_root.packageRoot, 'analyzer_plugin'));
-  await GeneratedContent.checkAll(pkg_root.packageRoot,
-      join(pkgPath, 'tool', 'spec', 'generate_all.dart'), allTargets);
+  await allTargets.check(
+    pkg_root.packageRoot,
+    join(pkgPath, 'tool', 'spec', 'generate_all.dart'),
+  );
 }
diff --git a/pkg/analyzer_testing/tool/api/generate_test.dart b/pkg/analyzer_testing/tool/api/generate_test.dart
@@ -6,13 +6,13 @@
 // date.
 
 import 'package:analyzer_testing/package_root.dart' as pkg_root;
-import 'package:analyzer_utilities/tools.dart';
+import 'package:analyzer_utilities/generated_content_check.dart';
 import 'package:path/path.dart';
 
 import 'generate.dart';
 
 Future<void> main() async {
-  await GeneratedContent.checkAll(
+  await allTargets.check(
     pkg_root.packageRoot,
     join(
       pkg_root.packageRoot,
@@ -21,6 +21,5 @@ Future<void> main() async {
       'api',
       'generate.dart',
     ),
-    allTargets,
   );
 }
diff --git a/pkg/analyzer_utilities/lib/generated_content_check.dart b/pkg/analyzer_utilities/lib/generated_content_check.dart
@@ -0,0 +1,134 @@
+import 'dart:async';
+import 'dart:io';
+
+import 'package:analyzer_utilities/tools.dart';
+import 'package:dart_style/dart_style.dart';
+import 'package:package_config/package_config.dart';
+import 'package:path/path.dart';
+import 'package:pub_semver/pub_semver.dart';
+import 'package:test/test.dart';
+
+Future<String> _formatText(String text, {required String pkgPath}) async {
+  var packageConfig = await findPackageConfig(Directory(pkgPath));
+  if (packageConfig == null) {
+    throw StateError(
+      'Could not find the shared Dart SDK package_config.json file, for '
+      '"$pkgPath"',
+    );
+  }
+  var package = packageConfig.packageOf(
+    Uri.file(join(pkgPath, 'pubspec.yaml')),
+  );
+  if (package == null) {
+    throw StateError('Could not find the package for "$pkgPath"');
+  }
+  var languageVersion = package.languageVersion;
+  if (languageVersion == null) {
+    throw StateError('Could not find a Dart language version for "$pkgPath"');
+  }
+  var version = Version(languageVersion.major, languageVersion.minor, 0);
+  return DartFormatter(languageVersion: version).format(text);
+}
+
+extension GeneratedContentExtension on GeneratedContent {
+  /// Check whether the [output] has the correct contents, and return true if it
+  /// does.
+  ///
+  /// [pkgRoot] is the path to the SDK's `pkg` directory.
+  Future<bool> check(String pkgRoot) async {
+    switch (this) {
+      case GeneratedDirectory self:
+        var outputDirectory = self.output(pkgRoot);
+        var map = self.directoryContentsComputer(pkgRoot);
+        try {
+          for (var entry in map.entries) {
+            var file = entry.key;
+            var fileContentsComputer = entry.value;
+            var expectedContents = await fileContentsComputer(pkgRoot);
+            var outputFile = File(posix.join(outputDirectory.path, file));
+            var actualContents = outputFile.readAsStringSync();
+            // Normalize Windows line endings to Unix line endings so that the
+            // comparison doesn't fail on Windows.
+            actualContents = actualContents.replaceAll('\r\n', '\n');
+            if (expectedContents != actualContents) {
+              return false;
+            }
+          }
+          var nonHiddenFileCount = 0;
+          outputDirectory
+              .listSync(recursive: false, followLinks: false)
+              .forEach((FileSystemEntity fileSystemEntity) {
+                if (fileSystemEntity is File &&
+                    !basename(fileSystemEntity.path).startsWith('.')) {
+                  nonHiddenFileCount++;
+                }
+              });
+          if (nonHiddenFileCount != map.length) {
+            // The number of files generated doesn't match the number we expected to
+            // generate.
+            return false;
+          }
+        } catch (e) {
+          // There was a problem reading the file (most likely because it didn't
+          // exist).  Treat that the same as if the file doesn't have the expected
+          // contents.
+          return false;
+        }
+        return true;
+      case GeneratedFile self:
+        var outputFile = self.output(pkgRoot);
+        var expectedContents = await self.computeContents(pkgRoot);
+        if (self.isDartFile) {
+          expectedContents = await _formatText(
+            expectedContents,
+            pkgPath: dirname(outputFile.path),
+          );
+        }
+        try {
+          var actualContents = outputFile.readAsStringSync();
+          // Normalize Windows line endings to Unix line endings so that the
+          // comparison doesn't fail on Windows.
+          actualContents = actualContents.replaceAll('\r\n', '\n');
+          expectedContents = expectedContents.replaceAll('\r\n', '\n');
+          return expectedContents == actualContents;
+        } catch (e) {
+          // There was a problem reading the file (most likely because it didn't
+          // exist).  Treat that the same as if the file doesn't have the expected
+          // contents.
+          return false;
+        }
+    }
+  }
+}
+
+extension GeneratedContentIterable on Iterable<GeneratedContent> {
+  /// Check that all of the targets in `this` are up to date.  If they are not,
+  /// print out a message instructing the user to regenerate them, and exit with
+  /// a nonzero error code.
+  ///
+  /// [pkgRoot] is the path to the SDK's `pkg` directory.  [generatorPath] is
+  /// the path to a .dart script the user may use to regenerate the targets.
+  ///
+  /// To avoid mistakes when run on Windows, [generatorPath] always uses
+  /// POSIX directory separators.
+  Future<void> check(String pkgRoot, String generatorPath) async {
+    var generateNeeded = false;
+    for (var target in this) {
+      var ok = await target.check(pkgRoot);
+      if (!ok) {
+        print(
+          '${normalize(target.output(pkgRoot).absolute.path)}'
+          " doesn't have expected contents.",
+        );
+        generateNeeded = true;
+      }
+    }
+    if (generateNeeded) {
+      print('Please regenerate using:');
+      var executable = Platform.executable;
+      var generateScript = normalize(joinAll(posix.split(generatorPath)));
+      print('  $executable $generateScript');
+      fail('Generated content needs to be regenerated');
+    }
+  }
+}
diff --git a/pkg/analyzer_utilities/lib/tools.dart b/pkg/analyzer_utilities/lib/tools.dart
diff --git a/pkg/linter/test/verify_generated_files_test.dart b/pkg/linter/test/verify_generated_files_test.dart
