DAS plugins: Cache AOT snapshot

Fixes #61684

This caching is based largely on the `--depfile` feature offered by
`dart compile`, which is based on a Ninja depfile concept
(https://ninja-build.org/manual.html#_depfile), which spits out a file
(`depfile.txt` here) which lists all of the input files which were
required to build an AOT snapshot.

The process is essentially:

1. If an AOT snapshot is found, maybe use it as a cached snapshot!
  a. If the `pubspec.yaml` modification timestamp is newer, re-compile!
  b. If the `.dart_tool/package_config.json` modification timestamp is
     newer, re-compile!
  c. If the `bin/plugin.dart` modification timestamp is newer,
     re-compile!
  d. If the `bin/depfile.txt` file is missing or malformed, re-compile!
  e. If any files mentioned in `bin/depfile.txt` have a newer
     modification timestamp, or don't exist, or are an otherwise bad
     path, re-compile!
  f. Otherwise, save a dozen seconds and use the cached snapshot.

Change-Id: Icc747198f8af76d256ac915685473d6f529a3cef
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/464602
Reviewed-by: Brian Wilkerson <[email protected]>
Commit-Queue: Samuel Rawlins <[email protected]>

Author: srawlins

pkg/analysis_server/lib/src/plugin/plugin_manager.dart

@@ -515,15 +515,16 @@ class PluginManager {
 
   /// Compiles [entrypoint] to an AOT snapshot and records timing to the
   /// instrumentation log.
-  ProcessResult _compileAotSnapshot(String entrypoint) {
+  ProcessResult _compileAotSnapshot(File entrypoint) {
     instrumentationService.logInfo(
       'Running "dart compile aot-snapshot $entrypoint".',
     );
 
     var stopwatch = Stopwatch()..start();
+    var depfile = entrypoint.parent.getChildAssumingFile('depfile.txt');
     var result = _processRunner.runSync(
       sdk.dart,
-      ['compile', 'aot-snapshot', entrypoint],
+      ['compile', 'aot-snapshot', '--depfile', depfile.path, entrypoint.path],
       stderrEncoding: utf8,
       stdoutEncoding: utf8,
     );
@@ -539,9 +540,30 @@ class PluginManager {
   /// Compiles [pluginFile], in [pluginFolder], to an AOT snapshot, and returns
   /// the [File] for the snapshot.
   File _compileAsAot({required File pluginFile, required Folder pluginFolder}) {
+    try {
+      // Potentially use existing snapshot.
+      var aotSnapshotFile = _existingAotSnapshot(
+        resourceProvider: _resourceProvider,
+        pluginFile: pluginFile,
+        pluginFolder: pluginFolder,
+      );
+      if (aotSnapshotFile != null) {
+        instrumentationService.logInfo(
+          'Using existing plugin AOT snapshot at '
+          "'${aotSnapshotFile.path}'",
+        );
+        return aotSnapshotFile;
+      }
+    } catch (error, stackTrace) {
+      instrumentationService.logException(
+        'Exception while checking an existing plugin AOT snapshot: '
+        '"$error"\n$stackTrace',
+      );
+    }
+
     // When the Dart Analysis Server is built as AOT, then all spawned
     // Isolates must also be built as AOT.
-    var aotResult = _compileAotSnapshot(pluginFile.path);
+    var aotResult = _compileAotSnapshot(pluginFile);
     if (aotResult.exitCode != 0) {
       var buffer = StringBuffer();
       buffer.writeln(
@@ -564,7 +586,7 @@ class PluginManager {
   /// Computes the plugin files, given that the plugin should exist in
   /// [pluginFolder].
   ///
-  /// Runs `pub` if [pubCommand] is not `null`.
+  /// Runs `pub <pubCommand>` in [pluginFolder] if [pubCommand] is not `null`.
   PluginFiles _computeFiles(
     Folder pluginFolder, {
     required bool builtAsAot,
@@ -582,7 +604,10 @@ class PluginManager {
         .getChildAssumingFile(file_paths.packageConfigJson);
 
     if (pubCommand != null) {
-      var pubResult = _runPubCommand(pubCommand, pluginFolder);
+      var pubResult = _runPubCommand(
+        pubCommand,
+        workingDirectory: pluginFolder,
+      );
       String? exceptionReason;
       if (pubResult.exitCode != 0) {
         var buffer = StringBuffer();
@@ -722,6 +747,56 @@ class PluginManager {
     return packageConfigFile;
   }
 
+  /// Returns a viable existing plugin AOT snapshot, if it exists and its
+  /// modification timestamp is newer than all of its dependencies, and `null`
+  /// otherwise.
+  ///
+  /// The dependencies of an AOT snapshot are the pubspec file, the
+  /// entrypoint file, and all of the files referenced in the depfile which
+  /// is generated by the `dart compile` command.
+  File? _existingAotSnapshot({
+    required ResourceProvider resourceProvider,
+    required File pluginFile,
+    required Folder pluginFolder,
+  }) {
+    var aotSnapshotFile = pluginFolder
+        .getChildAssumingFolder('bin')
+        .getChildAssumingFile('plugin.aot');
+    if (!aotSnapshotFile.exists) return null;
+    var snapshotModificationStamp = aotSnapshotFile.modificationStamp;
+
+    if (pluginFile.modificationStamp > snapshotModificationStamp) return null;
+    var pubspecFile = pluginFolder.getChildAssumingFile(file_paths.pubspecYaml);
+    if (pubspecFile.modificationStamp > snapshotModificationStamp) return null;
+
+    var depfile = pluginFolder
+        .getChildAssumingFolder('bin')
+        .getChildAssumingFile('depfile.txt');
+    if (!depfile.exists) return null;
+
+    var content = depfile.readAsStringSync();
+    var dependencies = parseDepfile(content);
+    if (dependencies == null) {
+      // Malformed depfile content.
+      return null;
+    }
+
+    for (var dependencyPath in dependencies) {
+      var file = _resourceProvider.getFile(dependencyPath);
+      if (!file.exists) {
+        // Something has certainly changed on disk; do not use the cached
+        // snapshot.
+        return null;
+      }
+      if (file.modificationStamp > snapshotModificationStamp) {
+        // Snapshot is stale.
+        return null;
+      }
+    }
+
+    return aotSnapshotFile;
+  }
+
   void _notifyPluginsChanged() => _pluginsChanged.add(null);
 
   /// Return the names of packages that are listed as dependencies in the given
@@ -742,17 +817,20 @@ class PluginManager {
   }
 
   /// Runs (and records timing to the instrumentation log) a Pub command
-  /// [pubCommand] in [folder].
-  ProcessResult _runPubCommand(String pubCommand, Folder folder) {
+  /// [pubCommand] in [workingDirectory].
+  ProcessResult _runPubCommand(
+    String pubCommand, {
+    required Folder workingDirectory,
+  }) {
     instrumentationService.logInfo(
-      'Running "pub $pubCommand" in "${folder.path}".',
+      'Running "pub $pubCommand" in "${workingDirectory.path}".',
     );
 
     var stopwatch = Stopwatch()..start();
     var result = _processRunner.runSync(
       sdk.dart,
       ['pub', pubCommand],
-      workingDirectory: folder.path,
+      workingDirectory: workingDirectory.path,
       environment: {_pubEnvironmentKey: _getPubEnvironmentValue()},
       stderrEncoding: utf8,
       stdoutEncoding: utf8,
@@ -772,6 +850,50 @@ class PluginManager {
     return hex.encode(bytes);
   }
 
+  /// Parses Ninja-style depfile content, returning a list of dependency paths.
+  ///
+  /// Returns `null` if the text is not valid depfile content.
+  ///
+  /// The format is:
+  ///
+  ///     target: dependency1 dependency2 ...
+  ///
+  /// See https://ninja-build.org/manual.html#_depfile.
+  @visibleForTesting
+  static List<String>? parseDepfile(String content) {
+    var colonIndex = content.indexOf(': ');
+    if (colonIndex < 0) {
+      // Not a valid depfile.
+      return null;
+    }
+    var dependenciesString = content
+        .substring(colonIndex + 1)
+        .trimLeft()
+        .replaceAll(RegExp(r'[\r\n]'), '');
+    var dependencies = <String>[];
+    var start = 0;
+    while (start < dependenciesString.length) {
+      var index = start;
+      while (index < dependenciesString.length) {
+        var char = dependenciesString[index];
+        if (char == ' ') {
+          break;
+        } else if (char == r'\') {
+          index++;
+        }
+        index++;
+      }
+      dependencies.add(
+        dependenciesString
+            .substring(start, index)
+            .replaceAll(r'\\', r'\')
+            .replaceAll(r'\ ', ' '),
+      );
+      start = index + 1;
+    }
+    return dependencies.where((p) => p.isNotEmpty).toList();
+  }
+
   /// Record the fact that the given [pluginIsolate] responded to a request with
   /// the given [method] in the given [time].
   static void recordResponseTime(

pkg/analysis_server/lib/src/plugin/plugin_watcher.dart

@@ -122,13 +122,26 @@ class PluginWatcher implements DriverWatcher {
       "context root: '${contextRoot.root.path}'",
     );
     sharedPluginFolder.create();
-    sharedPluginFolder
-        .getChildAssumingFile(file_paths.pubspecYaml)
-        .writeAsStringSync(packageGenerator.generatePubspec());
+    var pubspecFile = sharedPluginFolder.getChildAssumingFile(
+      file_paths.pubspecYaml,
+    );
+    var newPubspecContent = packageGenerator.generatePubspec();
+    // Only update the file if the content is different, to avoid changing the
+    // modification timestamp.
+    if (!pubspecFile.exists ||
+        newPubspecContent != pubspecFile.readAsStringSync()) {
+      pubspecFile.writeAsStringSync(newPubspecContent);
+    }
+
     var binFolder = sharedPluginFolder.getChildAssumingFolder('bin')..create();
-    binFolder
-        .getChildAssumingFile('plugin.dart')
-        .writeAsStringSync(packageGenerator.generateEntrypoint());
+    var entrypointFile = binFolder.getChildAssumingFile('plugin.dart');
+    var newEntrypointContent = packageGenerator.generateEntrypoint();
+    // Only update the file if the content is different, to avoid changing the
+    // modification timestamp.
+    if (!entrypointFile.exists ||
+        newEntrypointContent != entrypointFile.readAsStringSync()) {
+      entrypointFile.writeAsStringSync(newEntrypointContent);
+    }
     manager.instrumentationService.logInfo(
       'Adding ${pluginConfigurations.length} analyzer plugins for '
       "context root: '${contextRoot.root.path}'",