Lazily compute outputs. (#3960)

Author: davidmorgan

build_runner/CHANGELOG.md

@@ -13,6 +13,7 @@
 - Add `NodeType` to `AssetNode`, remove subtypes. Make mutations explicit.
 - Use `built_value` for `AssetNode` and related types.
 - Add details of what changed and what is built to `--verbose` logging.
+- Compute outputs as needed instead of storing them in the asset graph.
 
 ## 2.4.15
 

build_runner/bin/graph_inspector.dart

@@ -118,6 +118,7 @@ class InspectNodeCommand extends Command<bool> {
 
   @override
   bool run() {
+    var computedOutputs = assetGraph.computeOutputs();
     var argResults = this.argResults!;
     var stringUris = argResults.rest;
     if (stringUris.isEmpty) {
@@ -157,12 +158,16 @@ class InspectNodeCommand extends Command<bool> {
         node.primaryOutputs.forEach(printAsset);
 
         description.writeln('  secondary outputs:');
-        node.outputs.difference(node.primaryOutputs).forEach(printAsset);
+        (computedOutputs[node.id] ?? const <AssetId>{})
+            .difference(node.primaryOutputs.asSet())
+            .forEach(printAsset);
 
         if (node.type == NodeType.generated || node.type == NodeType.glob) {
           description.writeln('  inputs:');
           assetGraph.allNodes
-              .where((n) => n.outputs.contains(node.id))
+              .where(
+                (n) => (computedOutputs[n.id] ?? <AssetId>{}).contains(node.id),
+              )
               .map((n) => n.id)
               .forEach(printAsset);
         }

build_runner/lib/src/server/asset_graph_handler.dart

@@ -115,7 +115,8 @@ class AssetGraphHandler {
       {'id': '${node.id}', 'label': '${node.id}'},
     ];
     var edges = <Map<String, String>>[];
-    for (final output in node.outputs) {
+    var computedOutputs = _assetGraph.computeOutputs();
+    for (final output in (computedOutputs[node.id] ?? <AssetId>{})) {
       if (filterGlob != null && !filterGlob.matches(output.toString())) {
         continue;
       }

build_runner/test/generate/watch_test.dart

@@ -382,9 +382,6 @@ void main() {
           inputs: [makeAssetId('a|web/b.txt')],
           isHidden: false,
         );
-        builderOptionsNode = builderOptionsNode.rebuild(
-          (b) => b..outputs.add(bCopyNode.id),
-        );
         expectedGraph
           ..add(bCopyNode)
           ..add(
@@ -407,9 +404,6 @@ void main() {
           inputs: [makeAssetId('a|web/c.txt')],
           isHidden: false,
         );
-        builderOptionsNode = builderOptionsNode.rebuild(
-          (b) => b..outputs.add(cCopyNode.id),
-        );
         expectedGraph
           ..add(cCopyNode)
           ..add(

build_runner_core/CHANGELOG.md

@@ -29,6 +29,7 @@
 - Use `LibraryCycleGraphLoader` to load transitive deps for analysis.
 - Track post process builder outputs separately from the main graph Instead of
   in `postProcessAnchor` nodes.
+- Compute outputs as needed instead of storing them in the asset graph.
 
 ## 8.0.0
 

build_runner_core/lib/src/asset_graph/graph.dart

@@ -49,6 +49,10 @@ class AssetGraph implements GeneratedAssetHider {
 
   final BuiltMap<String, LanguageVersion?> packageLanguageVersions;
 
+  /// The result of [computeOutputs] for reuse, or `null` if outputs have not
+  /// been computed.
+  Map<AssetId, Set<AssetId>>? _outputs;
+
   /// All post process build steps outputs, indexed by package then
   /// [PostProcessBuildStepId].
   ///
@@ -129,6 +133,11 @@ class AssetGraph implements GeneratedAssetHider {
     if (node == null) throw StateError('Missing node: $id');
     final updatedNode = node.rebuild(updates);
     _nodesByPackage[id.package]![id.path] = updatedNode;
+
+    if (node.inputs != updatedNode.inputs) {
+      _outputs = null;
+    }
+
     return updatedNode;
   }
 
@@ -145,6 +154,11 @@ class AssetGraph implements GeneratedAssetHider {
     if (node == null) return null;
     final updatedNode = node.rebuild(updates);
     _nodesByPackage[id.package]![id.path] = updatedNode;
+
+    if (node.inputs != updatedNode.inputs) {
+      _outputs = null;
+    }
+
     return updatedNode;
   }
 
@@ -162,7 +176,6 @@ class AssetGraph implements GeneratedAssetHider {
         // primary outputs. We only want to remove this node.
         _nodesByPackage[existing.id.package]!.remove(existing.id.path);
         node = node.rebuild((b) {
-          b.outputs.addAll(existing.outputs);
           b.primaryOutputs.addAll(existing.primaryOutputs);
         });
       } else {
@@ -173,6 +186,10 @@ class AssetGraph implements GeneratedAssetHider {
       }
     }
     _nodesByPackage.putIfAbsent(node.id.package, () => {})[node.id.path] = node;
+    if (node.inputs?.isNotEmpty ?? false) {
+      _outputs = null;
+    }
+
     return node;
   }
 
@@ -261,7 +278,8 @@ class AssetGraph implements GeneratedAssetHider {
     for (var output in node.primaryOutputs.toList()) {
       _removeRecursive(output, removedIds: removedIds);
     }
-    for (var output in node.outputs) {
+    final outputs = computeOutputs();
+    for (var output in (outputs[node.id] ?? const <AssetId>{})) {
       updateNodeIfPresent(output, (nodeBuilder) {
         if (nodeBuilder.type == NodeType.generated) {
           nodeBuilder.generatedNodeState.inputs.remove(id);
@@ -277,23 +295,14 @@ class AssetGraph implements GeneratedAssetHider {
       for (var input in node.generatedNodeState!.inputs) {
         // We may have already removed this node entirely.
         updateNodeIfPresent(input, (nodeBuilder) {
-          nodeBuilder
-            ..outputs.remove(id)
-            ..primaryOutputs.remove(id);
+          nodeBuilder.primaryOutputs.remove(id);
         });
       }
-      updateNode(node.generatedNodeConfiguration!.builderOptionsId, (
-        nodeBuilder,
-      ) {
-        nodeBuilder.outputs.remove(id);
-      });
     } else if (node.type == NodeType.glob) {
       for (var input in node.globNodeState!.inputs) {
         // We may have already removed this node entirely.
         updateNodeIfPresent(input, (nodeBuilder) {
-          nodeBuilder
-            ..outputs.remove(id)
-            ..primaryOutputs.remove(id);
+          nodeBuilder.primaryOutputs.remove(id);
         });
       }
     }
@@ -311,6 +320,34 @@ class AssetGraph implements GeneratedAssetHider {
     return removedIds;
   }
 
+  /// Computes node outputs: the inverse of the graph described by the `inputs`
+  /// fields on glob and generated nodes.
+  ///
+  /// The result is cached until any node is updated with different `inputs` or
+  /// [updateAndInvalidate] is called.
+  Map<AssetId, Set<AssetId>> computeOutputs() {
+    if (_outputs != null) return _outputs!;
+    final result = <AssetId, Set<AssetId>>{};
+    for (final node in allNodes) {
+      if (node.type == NodeType.generated) {
+        for (final input in node.generatedNodeState!.inputs) {
+          result.putIfAbsent(input, () => {}).add(node.id);
+        }
+        result
+            .putIfAbsent(
+              node.generatedNodeConfiguration!.builderOptionsId,
+              () => {},
+            )
+            .add(node.id);
+      } else if (node.type == NodeType.glob) {
+        for (final input in node.globNodeState!.inputs) {
+          result.putIfAbsent(input, () => {}).add(node.id);
+        }
+      }
+    }
+    return _outputs = result;
+  }
+
   /// All nodes in the graph, whether source files or generated outputs.
   Iterable<AssetNode> get allNodes =>
       _nodesByPackage.values.expand((pkdIds) => pkdIds.values);
@@ -408,8 +445,7 @@ class AssetGraph implements GeneratedAssetHider {
           .where(
             (node) =>
                 node.isTrackedInput &&
-                (node.outputs.isNotEmpty ||
-                    node.primaryOutputs.isNotEmpty ||
+                (node.primaryOutputs.isNotEmpty ||
                     node.lastKnownDigest != null),
           )
           .map((node) => node.id),
@@ -458,6 +494,7 @@ class AssetGraph implements GeneratedAssetHider {
     // Transitively invalidates all assets. This needs to happen after the
     // structure of the graph has been updated.
     var invalidatedIds = <AssetId>{};
+    final computedOutputs = computeOutputs();
 
     void invalidateNodeAndDeps(AssetId startNodeId) {
       if (!invalidatedIds.add(startNodeId)) return;
@@ -484,7 +521,8 @@ class AssetGraph implements GeneratedAssetHider {
           });
 
           if (invalidatedNode != null) {
-            for (final id in invalidatedNode.outputs) {
+            for (final id
+                in (computedOutputs[invalidatedNode.id] ?? const <AssetId>{})) {
               if (invalidatedIds.add(id)) {
                 nodesToInvalidate.add(id);
               }
@@ -534,6 +572,7 @@ class AssetGraph implements GeneratedAssetHider {
       }
     }
 
+    _outputs = null;
     return invalidatedIds;
   }
 
@@ -681,12 +720,14 @@ class AssetGraph implements GeneratedAssetHider {
       throw ArgumentError('Expected node of type NodeType.builderOptionsNode');
     }
     var removed = <AssetId>{};
+    Map<AssetId, Set<AssetId>>? computedOutputsBeforeRemoves;
     for (var output in outputs) {
       AssetNode? existing;
       // When any outputs aren't hidden we can pick up old generated outputs as
       // regular `AssetNode`s, we need to delete them and all their primary
       // outputs, and replace them with a `GeneratedAssetNode`.
       if (contains(output)) {
+        computedOutputsBeforeRemoves = computeOutputs();
         existing = get(output)!;
         if (existing.type == NodeType.generated) {
           final existingConfiguration = existing.generatedNodeConfiguration!;
@@ -712,13 +753,9 @@ class AssetGraph implements GeneratedAssetHider {
         isHidden: isHidden,
       );
       if (existing != null) {
-        newNode = newNode.rebuild((b) => b..outputs.addAll(existing!.outputs));
         // Ensure we set up the reverse link for NodeWithInput nodes.
-        _addInput(existing.outputs, output);
+        _addInput(computedOutputsBeforeRemoves![output] ?? <AssetId>{}, output);
       }
-      updateNode(builderOptionsNode.id, (nodeBuilder) {
-        nodeBuilder.outputs.add(output);
-      });
       _add(newNode);
     }
     return removed;

build_runner_core/lib/src/asset_graph/node.dart

@@ -57,10 +57,6 @@ abstract class AssetNode implements Built<AssetNode, AssetNodeBuilder> {
   /// when run on this asset.
   BuiltSet<AssetId> get primaryOutputs;
 
-  /// The [AssetId]s of all generated assets which are output by a [Builder]
-  /// which reads this asset.
-  BuiltSet<AssetId> get outputs;
-
   /// The [Digest] for this node in its last known state.
   ///
   /// May be `null` if this asset has no outputs, or if it doesn't actually
@@ -96,7 +92,6 @@ abstract class AssetNode implements Built<AssetNode, AssetNodeBuilder> {
   bool get changesRequireRebuild =>
       type == NodeType.internal ||
       type == NodeType.glob ||
-      outputs.isNotEmpty ||
       lastKnownDigest != null;
 
   factory AssetNode([void Function(AssetNodeBuilder) updates]) = _$AssetNode;
@@ -124,7 +119,6 @@ abstract class AssetNode implements Built<AssetNode, AssetNodeBuilder> {
     b.id = id;
     b.type = NodeType.source;
     b.primaryOutputs.replace(primaryOutputs ?? {});
-    b.outputs.replace(outputs ?? {});
     b.lastKnownDigest = lastKnownDigest;
   });
 
@@ -239,6 +233,19 @@ abstract class AssetNode implements Built<AssetNode, AssetNodeBuilder> {
       globNodeState != null,
     );
   }
+
+  /// The generated node inputs, or the glob node inputs, or `null` if the node
+  /// is not of one of those two types.
+  BuiltSet<AssetId>? get inputs {
+    switch (type) {
+      case NodeType.generated:
+        return generatedNodeState!.inputs;
+      case NodeType.glob:
+        return globNodeState!.inputs;
+      default:
+        return null;
+    }
+  }
 }
 
 /// Additional configuration for an [AssetNode.generated].