Skip to content

Commit 464e9f2

Browse files
davidmorgandavidmorgan
authored
Serialize AssetId by identity. (#3970)
1 parent a5f7a85 commit 464e9f2

File tree

4 files changed

+177
-133
lines changed

4 files changed

+177
-133
lines changed

‎build_runner_core/lib/src/asset_graph/graph.dart‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ class AssetGraph implements GeneratedAssetHider {
7070

7171
/// Deserializes this graph.
7272
factory AssetGraph.deserialize(List<int> serializedGraph) =>
73-
_AssetGraphDeserializer(serializedGraph).deserialize();
73+
deserializeAssetGraph(serializedGraph);
7474

7575
static Future<AssetGraph> build(
7676
BuildPhases buildPhases,
@@ -106,7 +106,7 @@ class AssetGraph implements GeneratedAssetHider {
106106
return graph;
107107
}
108108

109-
List<int> serialize() => _AssetGraphSerializer(this).serialize();
109+
List<int> serialize() => serializeAssetGraph(this);
110110

111111
@visibleForTesting
112112
Map<String, Map<PostProcessBuildStepId, Set<AssetId>>>

‎build_runner_core/lib/src/asset_graph/identity_serializer.dart‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import 'package:built_value/serializer.dart';
6+
7+
/// Wraps another `built_value` serializer to add serializing by identity.
8+
///
9+
/// Integer IDs are created as new values are encountered, and a mapping to
10+
/// objects and serialized object values is maintained.
11+
///
12+
/// The serialized data does not contain the object values, only IDs. This might
13+
/// not matter if this `IdentitySerializer` will stay in memory and be used to
14+
/// deserialize. Or, the serialized object values must be serialized separately:
15+
/// they can be obtained from [serializedObjects], and set via
16+
/// [deserializeWithObjects].
17+
///
18+
/// TODO(davidmorgan): consider upstreaming to `built_value`.
19+
class IdentitySerializer<T> implements PrimitiveSerializer<T> {
20+
final Serializer<T> delegate;
21+
final PrimitiveSerializer<T>? _primitiveDelegate;
22+
final StructuredSerializer<T>? _structuredDelegate;
23+
24+
final Map<T, int> _ids = Map.identity();
25+
final List<T> _objects = [];
26+
List<Object?> _serializedObjects = [];
27+
28+
/// A serializer wrapping [delegate] to deduplicate by identity.
29+
IdentitySerializer(this.delegate)
30+
: _primitiveDelegate = delegate is PrimitiveSerializer<T> ? delegate : null,
31+
_structuredDelegate =
32+
delegate is StructuredSerializer<T> ? delegate : null;
33+
34+
/// Sets the stored object values to [objects].
35+
///
36+
/// Serialized values are indices into this list.
37+
void deserializeWithObjects(Iterable<T> objects) {
38+
_ids.clear();
39+
_objects.clear();
40+
_serializedObjects.clear();
41+
for (final object in objects) {
42+
_objects.add(object);
43+
_ids[object] = _objects.length - 1;
44+
}
45+
}
46+
47+
/// The list of unique objects encountered since the most recent [reset].
48+
List<Object?> get serializedObjects => _serializedObjects;
49+
50+
/// Clears the ID to object and serialized object mappings.
51+
///
52+
/// Call this after serializing or deserializing to avoid retaining objects in
53+
/// memory; or, don't call it to continue using the same IDs and objects.
54+
void reset() {
55+
_ids.clear();
56+
_objects.clear();
57+
_serializedObjects = [];
58+
}
59+
60+
@override
61+
Iterable<Type> get types => delegate.types;
62+
63+
@override
64+
String get wireName => delegate.wireName;
65+
66+
@override
67+
T deserialize(
68+
Serializers serializers,
69+
Object serialized, {
70+
FullType specifiedType = FullType.unspecified,
71+
}) => _objects[serialized as int];
72+
73+
@override
74+
Object serialize(
75+
Serializers serializers,
76+
T object, {
77+
FullType specifiedType = FullType.unspecified,
78+
}) {
79+
// If it has already been seen, return the ID.
80+
return _ids.putIfAbsent(object, () {
81+
// Otherwise, serialize it, store the value and serialized value, and
82+
// return the index of the last of `_objects` as the ID.
83+
final serialized =
84+
_primitiveDelegate == null
85+
? _structuredDelegate!.serialize(serializers, object)
86+
: _primitiveDelegate.serialize(serializers, object);
87+
_serializedObjects.add(serialized);
88+
return (_objects..add(object)).length - 1;
89+
});
90+
}
91+
}

‎build_runner_core/lib/src/asset_graph/serialization.dart‎

Lines changed: 77 additions & 130 deletions
Original file line numberDiff line numberDiff line change
@@ -8,148 +8,95 @@ part of 'graph.dart';
88
///
99
/// This should be incremented any time the serialize/deserialize formats
1010
/// change.
11-
const _version = 27;
11+
const _version = 28;
1212

1313
/// Deserializes an [AssetGraph] from a [Map].
14-
class _AssetGraphDeserializer {
15-
// Iteration order does not matter
16-
final _idToAssetId = HashMap<int, AssetId>();
17-
final Map _serializedGraph;
18-
19-
_AssetGraphDeserializer._(this._serializedGraph);
20-
21-
factory _AssetGraphDeserializer(List<int> bytes) {
22-
dynamic decoded;
23-
try {
24-
decoded = jsonDecode(utf8.decode(bytes));
25-
} on FormatException {
26-
throw AssetGraphCorruptedException();
27-
}
28-
if (decoded is! Map) throw AssetGraphCorruptedException();
29-
if (decoded['version'] != _version) {
30-
throw AssetGraphCorruptedException();
31-
}
32-
return _AssetGraphDeserializer._(decoded);
14+
AssetGraph deserializeAssetGraph(List<int> bytes) {
15+
dynamic serializedGraph;
16+
try {
17+
serializedGraph = jsonDecode(utf8.decode(bytes));
18+
} on FormatException {
19+
throw AssetGraphCorruptedException();
20+
}
21+
if (serializedGraph is! Map) throw AssetGraphCorruptedException();
22+
if (serializedGraph['version'] != _version) {
23+
throw AssetGraphCorruptedException();
3324
}
3425

35-
/// Perform the deserialization, should only be called once.
36-
AssetGraph deserialize() {
37-
var packageLanguageVersions = {
38-
for (var entry
39-
in (_serializedGraph['packageLanguageVersions']
40-
as Map<String, dynamic>)
41-
.entries)
42-
entry.key:
43-
entry.value != null
44-
? LanguageVersion.parse(entry.value as String)
45-
: null,
46-
};
47-
var graph = AssetGraph._(
48-
_deserializeDigest(_serializedGraph['buildActionsDigest'] as String)!,
49-
_serializedGraph['dart_version'] as String,
50-
packageLanguageVersions.build(),
51-
BuiltList<String>.from(_serializedGraph['enabledExperiments'] as List),
52-
);
53-
54-
var packageNames = _serializedGraph['packages'] as List;
55-
56-
// Read in the id => AssetId map from the graph first.
57-
var assetPaths = _serializedGraph['assetPaths'] as List;
58-
for (var i = 0; i < assetPaths.length; i += 2) {
59-
var packageName = packageNames[assetPaths[i + 1] as int] as String;
60-
_idToAssetId[i ~/ 2] = AssetId(packageName, assetPaths[i] as String);
61-
}
62-
63-
// Read in all the nodes and their outputs.
64-
//
65-
// Note that this does not read in the inputs of generated nodes.
66-
for (var serializedItem in _serializedGraph['nodes'] as Iterable) {
67-
graph._add(_deserializeAssetNode(serializedItem as List));
68-
}
26+
identityAssetIdSerializer.deserializeWithObjects(
27+
(serializedGraph['ids'] as List).map(
28+
(id) => assetIdSerializer.deserialize(serializers, id as Object),
29+
),
30+
);
31+
32+
var packageLanguageVersions = {
33+
for (var entry
34+
in (serializedGraph['packageLanguageVersions'] as Map<String, dynamic>)
35+
.entries)
36+
entry.key:
37+
entry.value != null
38+
? LanguageVersion.parse(entry.value as String)
39+
: null,
40+
};
41+
var graph = AssetGraph._(
42+
_deserializeDigest(serializedGraph['buildActionsDigest'] as String)!,
43+
serializedGraph['dart_version'] as String,
44+
packageLanguageVersions.build(),
45+
BuiltList<String>.from(serializedGraph['enabledExperiments'] as List),
46+
);
47+
48+
for (var serializedItem in serializedGraph['nodes'] as Iterable) {
49+
graph._add(_deserializeAssetNode(serializedItem as List));
50+
}
6951

70-
final postProcessOutputs =
71-
serializers.deserialize(
72-
_serializedGraph['postProcessOutputs'],
73-
specifiedType: postProcessBuildStepOutputsFullType,
74-
)
75-
as Map<String, Map<PostProcessBuildStepId, Set<AssetId>>>;
52+
final postProcessOutputs =
53+
serializers.deserialize(
54+
serializedGraph['postProcessOutputs'],
55+
specifiedType: postProcessBuildStepOutputsFullType,
56+
)
57+
as Map<String, Map<PostProcessBuildStepId, Set<AssetId>>>;
7658

77-
for (final postProcessOutputsForPackage in postProcessOutputs.values) {
78-
for (final entry in postProcessOutputsForPackage.entries) {
79-
graph.updatePostProcessBuildStep(entry.key, outputs: entry.value);
80-
}
59+
for (final postProcessOutputsForPackage in postProcessOutputs.values) {
60+
for (final entry in postProcessOutputsForPackage.entries) {
61+
graph.updatePostProcessBuildStep(entry.key, outputs: entry.value);
8162
}
82-
83-
return graph;
8463
}
8564

86-
AssetNode _deserializeAssetNode(List serializedNode) =>
87-
serializers.deserializeWith(AssetNode.serializer, serializedNode)
88-
as AssetNode;
65+
identityAssetIdSerializer.reset();
66+
return graph;
8967
}
9068

91-
/// Serializes an [AssetGraph] into a [Map].
92-
class _AssetGraphSerializer {
93-
// Iteration order does not matter
94-
final _assetIdToId = HashMap<AssetId, int>();
95-
96-
final AssetGraph _graph;
97-
98-
_AssetGraphSerializer(this._graph);
99-
100-
/// Perform the serialization, should only be called once.
101-
List<int> serialize() {
102-
var pathId = 0;
103-
// [path0, packageId0, path1, packageId1, ...]
104-
var assetPaths = <dynamic>[];
105-
var packages = _graph._nodesByPackage.keys.toList(growable: false);
106-
for (var node in _graph.allNodes) {
107-
_assetIdToId[node.id] = pathId;
108-
pathId++;
109-
assetPaths
110-
..add(node.id.path)
111-
..add(packages.indexOf(node.id.package));
112-
}
69+
AssetNode _deserializeAssetNode(List serializedNode) =>
70+
serializers.deserializeWith(AssetNode.serializer, serializedNode)
71+
as AssetNode;
11372

114-
var result = <String, dynamic>{
115-
'version': _version,
116-
'dart_version': _graph.dartVersion,
117-
'nodes': _graph.allNodes
118-
.map((node) => serializers.serializeWith(AssetNode.serializer, node))
119-
.toList(growable: false),
120-
'buildActionsDigest': _serializeDigest(_graph.buildPhasesDigest),
121-
'packages': packages,
122-
'assetPaths': assetPaths,
123-
'packageLanguageVersions':
124-
_graph.packageLanguageVersions
125-
.map((pkg, version) => MapEntry(pkg, version?.toString()))
126-
.toMap(),
127-
'enabledExperiments': _graph.enabledExperiments.toList(),
128-
'postProcessOutputs': serializers.serialize(
129-
_graph._postProcessBuildStepOutputs,
130-
specifiedType: postProcessBuildStepOutputsFullType,
131-
),
132-
};
133-
return utf8.encode(json.encode(result));
134-
}
135-
136-
int findAssetIndex(
137-
AssetId id, {
138-
required AssetId from,
139-
required String field,
140-
}) {
141-
final index = _assetIdToId[id];
142-
if (index == null) {
143-
log.severe(
144-
'The $field field in $from references a non-existent asset '
145-
'$id and will corrupt the asset graph. '
146-
'If you encounter this error please copy '
147-
'the details from this message and add them to '
148-
'https://github.com/dart-lang/build/issues/1804.',
149-
);
150-
}
151-
return index!;
152-
}
73+
/// Serializes an [AssetGraph] into a [Map].
74+
List<int> serializeAssetGraph(AssetGraph graph) {
75+
// Serialize nodes first so all `AssetId` instances are seen by
76+
// `identityAssetIdSeralizer`.
77+
final nodes = graph.allNodes
78+
.map((node) => serializers.serializeWith(AssetNode.serializer, node))
79+
.toList(growable: false);
80+
81+
var result = <String, dynamic>{
82+
'version': _version,
83+
'ids': identityAssetIdSerializer.serializedObjects,
84+
'dart_version': graph.dartVersion,
85+
'nodes': nodes,
86+
'buildActionsDigest': _serializeDigest(graph.buildPhasesDigest),
87+
'packageLanguageVersions':
88+
graph.packageLanguageVersions
89+
.map((pkg, version) => MapEntry(pkg, version?.toString()))
90+
.toMap(),
91+
'enabledExperiments': graph.enabledExperiments.toList(),
92+
'postProcessOutputs': serializers.serialize(
93+
graph._postProcessBuildStepOutputs,
94+
specifiedType: postProcessBuildStepOutputsFullType,
95+
),
96+
};
97+
98+
identityAssetIdSerializer.reset();
99+
return utf8.encode(json.encode(result));
153100
}
154101

155102
Digest? _deserializeDigest(String? serializedDigest) =>

‎build_runner_core/lib/src/asset_graph/serializers.dart‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import 'package:built_collection/built_collection.dart';
99
import 'package:built_value/serializer.dart';
1010
import 'package:crypto/crypto.dart';
1111

12+
import 'identity_serializer.dart';
1213
import 'node.dart';
1314
import 'post_process_build_step_id.dart';
1415

@@ -24,10 +25,15 @@ final postProcessBuildStepOutputsFullType = FullType(Map, [
2425
postProcessBuildStepOutputsInnerFullType,
2526
]);
2627

28+
final assetIdSerializer = AssetIdSerializer();
29+
final identityAssetIdSerializer = IdentitySerializer<AssetId>(
30+
assetIdSerializer,
31+
);
32+
2733
@SerializersFor([AssetNode])
2834
final Serializers serializers =
2935
(_$serializers.toBuilder()
30-
..add(AssetIdSerializer())
36+
..add(identityAssetIdSerializer)
3137
..add(DigestSerializer())
3238
..add(MapSerializer())
3339
..add(SetSerializer())

0 commit comments

Comments
 (0)