Skip to content

Commit 2a437c5

Browse files
mralephCommit Queue
mraleph
authored and
Commit Queue
committed
[vm] Intern strings when writing Perfetto timeline
Intern the following fields: * Category names * Event labels * Debug annotation keys and values This significantly reduces the size of the timeline (e.g. a timeline containing 60k slices goes from 15Mb to 5Mb timeline) This relands commit f2614d2 with a fix for Android build. TEST=ci and manually Cq-Include-Trybots: luci.dart.try:vm-ffi-android-release-arm64c-try Change-Id: I88d4c5e1142ff66b270a22b82bacd1e9313fa953 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/418220 Reviewed-by: Derek Xu <[email protected]>
1 parent c0b423f commit 2a437c5

20 files changed

+1515
-171
lines changed

‎pkg/vm_service/test/get_perfetto_vm_timeline_rpc_test.dart‎

Lines changed: 105 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
11
// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
22
// for details. All rights reserved. Use of this source code is governed by a
33
// BSD-style license that can be found in the LICENSE file.
4+
//
5+
// VMOptions=
6+
// VMOptions=--intern_strings_when_writing_perfetto_timeline
47

58
import 'dart:collection';
69
import 'dart:convert';
710
import 'dart:developer';
11+
import 'dart:io' show Platform;
812

913
import 'package:test/test.dart';
1014
import 'package:vm_service/vm_service.dart' hide Timeline;
@@ -35,12 +39,108 @@ void primeTimeline() {
3539
Timeline.finishSync();
3640
}
3741

38-
Iterable<TrackEvent> extractTrackEventsFromTracePackets(
42+
class Deinterner {
43+
final bool stringsShouldBeInterned = Platform.executableArguments
44+
.contains('--intern_strings_when_writing_perfetto_timeline');
45+
46+
final Map<int, String> debugAnnotationNames = {};
47+
final Map<int, String> debugAnnotationStringValues = {};
48+
final Map<int, String> eventNames = {};
49+
final Map<int, String> eventCategories = {};
50+
51+
/// Update the state of the interning dictionaries using [InternedData]
52+
/// from the given packet.
53+
void update(TracePacket packet) {
54+
// Clear the state if [TracePacket.sequenceFlags] instructs us to do so.
55+
if (packet.sequenceFlags &
56+
TracePacket_SequenceFlags.SEQ_INCREMENTAL_STATE_CLEARED.value !=
57+
0) {
58+
debugAnnotationNames.clear();
59+
debugAnnotationStringValues.clear();
60+
eventNames.clear();
61+
eventCategories.clear();
62+
}
63+
64+
if (!packet.hasInternedData()) {
65+
return;
66+
}
67+
68+
final internedData = packet.internedData;
69+
for (var e in internedData.debugAnnotationNames) {
70+
debugAnnotationNames[e.iid.toInt()] = e.name;
71+
}
72+
for (var e in internedData.debugAnnotationStringValues) {
73+
debugAnnotationStringValues[e.iid.toInt()] = utf8.decode(e.str);
74+
}
75+
for (var e in internedData.eventNames) {
76+
eventNames[e.iid.toInt()] = e.name;
77+
}
78+
for (var e in internedData.eventCategories) {
79+
eventCategories[e.iid.toInt()] = e.name;
80+
}
81+
}
82+
83+
/// Deintern contents of the given [TrackEvent].
84+
void deintern(TrackEvent event) {
85+
if (event.hasName()) {
86+
expect(stringsShouldBeInterned, isFalse);
87+
}
88+
if (event.hasNameIid()) {
89+
expect(stringsShouldBeInterned, isTrue);
90+
expect(event.hasName(), isFalse);
91+
event.name = eventNames[event.nameIid.toInt()]!;
92+
event.clearNameIid();
93+
}
94+
95+
if (event.categories.isNotEmpty) {
96+
expect(stringsShouldBeInterned, isFalse);
97+
}
98+
if (event.categoryIids.isNotEmpty) {
99+
expect(stringsShouldBeInterned, isTrue);
100+
expect(event.categories.isEmpty, isTrue);
101+
for (var iid in event.categoryIids) {
102+
event.categories.add(eventCategories[iid.toInt()]!);
103+
}
104+
event.categoryIids.clear();
105+
}
106+
for (var annotation in event.debugAnnotations) {
107+
if (annotation.hasStringValue()) {
108+
expect(stringsShouldBeInterned, isFalse);
109+
}
110+
if (annotation.hasStringValueIid()) {
111+
expect(stringsShouldBeInterned, isTrue);
112+
expect(annotation.hasStringValue(), isFalse);
113+
annotation.stringValue =
114+
debugAnnotationStringValues[annotation.stringValueIid.toInt()]!;
115+
annotation.clearStringValueIid();
116+
}
117+
118+
if (annotation.hasName()) {
119+
expect(stringsShouldBeInterned, isFalse);
120+
}
121+
if (annotation.hasNameIid()) {
122+
expect(stringsShouldBeInterned, isTrue);
123+
expect(annotation.hasName(), isFalse);
124+
annotation.name = debugAnnotationNames[annotation.nameIid.toInt()]!;
125+
annotation.clearNameIid();
126+
}
127+
}
128+
}
129+
}
130+
131+
List<TrackEvent> extractTrackEventsFromTracePackets(
39132
List<TracePacket> packets,
40133
) {
41-
return packets
42-
.where((packet) => packet.hasTrackEvent())
43-
.map((packet) => packet.trackEvent);
134+
final result = <TrackEvent>[];
135+
final deinterner = Deinterner();
136+
for (var packet in packets) {
137+
deinterner.update(packet);
138+
if (packet.hasTrackEvent()) {
139+
deinterner.deintern(packet.trackEvent);
140+
result.add(packet.trackEvent);
141+
}
142+
}
143+
return result;
44144
}
45145

46146
Map<String, String> mapFromListOfDebugAnnotations(
@@ -60,7 +160,7 @@ Map<String, String> mapFromListOfDebugAnnotations(
60160
}
61161

62162
void checkThatAllEventsHaveIsolateNumbers(Iterable<TrackEvent> events) {
63-
for (TrackEvent event in events) {
163+
for (final event in events) {
64164
final debugAnnotations =
65165
mapFromListOfDebugAnnotations(event.debugAnnotations);
66166
expect(debugAnnotations['isolateGroupId'], isNotNull);

‎pkg/vm_service_protos/lib/src/protos/perfetto/trace/interned_data/interned_data.pb.dart‎

Lines changed: 66 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,9 @@ import 'dart:core' as $core;
2121

2222
import 'package:protobuf/protobuf.dart' as $pb;
2323

24-
import '../profiling/profile_common.pb.dart' as $2;
24+
import '../profiling/profile_common.pb.dart' as $3;
25+
import '../track_event/debug_annotation.pb.dart' as $1;
26+
import '../track_event/track_event.pb.dart' as $2;
2527

2628
export 'package:protobuf/protobuf.dart' show GeneratedMessageGenericExtensions;
2729

@@ -36,13 +38,26 @@ export 'package:protobuf/protobuf.dart' show GeneratedMessageGenericExtensions;
3638
/// Next id: 29.
3739
class InternedData extends $pb.GeneratedMessage {
3840
factory InternedData({
39-
$core.Iterable<$2.InternedString>? functionNames,
40-
$core.Iterable<$2.Frame>? frames,
41-
$core.Iterable<$2.Callstack>? callstacks,
42-
$core.Iterable<$2.InternedString>? mappingPaths,
43-
$core.Iterable<$2.Mapping>? mappings,
41+
$core.Iterable<$2.EventCategory>? eventCategories,
42+
$core.Iterable<$2.EventName>? eventNames,
43+
$core.Iterable<$1.DebugAnnotationName>? debugAnnotationNames,
44+
$core.Iterable<$3.InternedString>? functionNames,
45+
$core.Iterable<$3.Frame>? frames,
46+
$core.Iterable<$3.Callstack>? callstacks,
47+
$core.Iterable<$3.InternedString>? mappingPaths,
48+
$core.Iterable<$3.Mapping>? mappings,
49+
$core.Iterable<$3.InternedString>? debugAnnotationStringValues,
4450
}) {
4551
final $result = create();
52+
if (eventCategories != null) {
53+
$result.eventCategories.addAll(eventCategories);
54+
}
55+
if (eventNames != null) {
56+
$result.eventNames.addAll(eventNames);
57+
}
58+
if (debugAnnotationNames != null) {
59+
$result.debugAnnotationNames.addAll(debugAnnotationNames);
60+
}
4661
if (functionNames != null) {
4762
$result.functionNames.addAll(functionNames);
4863
}
@@ -58,6 +73,9 @@ class InternedData extends $pb.GeneratedMessage {
5873
if (mappings != null) {
5974
$result.mappings.addAll(mappings);
6075
}
76+
if (debugAnnotationStringValues != null) {
77+
$result.debugAnnotationStringValues.addAll(debugAnnotationStringValues);
78+
}
6179
return $result;
6280
}
6381
InternedData._() : super();
@@ -73,19 +91,33 @@ class InternedData extends $pb.GeneratedMessage {
7391
package:
7492
const $pb.PackageName(_omitMessageNames ? '' : 'perfetto.protos'),
7593
createEmptyInstance: create)
76-
..pc<$2.InternedString>(
94+
..pc<$2.EventCategory>(
95+
1, _omitFieldNames ? '' : 'eventCategories', $pb.PbFieldType.PM,
96+
subBuilder: $2.EventCategory.create)
97+
..pc<$2.EventName>(
98+
2, _omitFieldNames ? '' : 'eventNames', $pb.PbFieldType.PM,
99+
subBuilder: $2.EventName.create)
100+
..pc<$1.DebugAnnotationName>(
101+
3, _omitFieldNames ? '' : 'debugAnnotationNames', $pb.PbFieldType.PM,
102+
subBuilder: $1.DebugAnnotationName.create)
103+
..pc<$3.InternedString>(
77104
5, _omitFieldNames ? '' : 'functionNames', $pb.PbFieldType.PM,
78-
subBuilder: $2.InternedString.create)
79-
..pc<$2.Frame>(6, _omitFieldNames ? '' : 'frames', $pb.PbFieldType.PM,
80-
subBuilder: $2.Frame.create)
81-
..pc<$2.Callstack>(
105+
subBuilder: $3.InternedString.create)
106+
..pc<$3.Frame>(6, _omitFieldNames ? '' : 'frames', $pb.PbFieldType.PM,
107+
subBuilder: $3.Frame.create)
108+
..pc<$3.Callstack>(
82109
7, _omitFieldNames ? '' : 'callstacks', $pb.PbFieldType.PM,
83-
subBuilder: $2.Callstack.create)
84-
..pc<$2.InternedString>(
110+
subBuilder: $3.Callstack.create)
111+
..pc<$3.InternedString>(
85112
17, _omitFieldNames ? '' : 'mappingPaths', $pb.PbFieldType.PM,
86-
subBuilder: $2.InternedString.create)
87-
..pc<$2.Mapping>(19, _omitFieldNames ? '' : 'mappings', $pb.PbFieldType.PM,
88-
subBuilder: $2.Mapping.create)
113+
subBuilder: $3.InternedString.create)
114+
..pc<$3.Mapping>(19, _omitFieldNames ? '' : 'mappings', $pb.PbFieldType.PM,
115+
subBuilder: $3.Mapping.create)
116+
..pc<$3.InternedString>(
117+
29,
118+
_omitFieldNames ? '' : 'debugAnnotationStringValues',
119+
$pb.PbFieldType.PM,
120+
subBuilder: $3.InternedString.create)
89121
..hasRequiredFields = false;
90122

91123
@$core.Deprecated('Using this can add significant overhead to your binary. '
@@ -111,25 +143,38 @@ class InternedData extends $pb.GeneratedMessage {
111143
$pb.GeneratedMessage.$_defaultFor<InternedData>(create);
112144
static InternedData? _defaultInstance;
113145

146+
@$pb.TagNumber(1)
147+
$pb.PbList<$2.EventCategory> get eventCategories => $_getList(0);
148+
149+
@$pb.TagNumber(2)
150+
$pb.PbList<$2.EventName> get eventNames => $_getList(1);
151+
152+
@$pb.TagNumber(3)
153+
$pb.PbList<$1.DebugAnnotationName> get debugAnnotationNames => $_getList(2);
154+
114155
/// Names of functions used in frames below.
115156
@$pb.TagNumber(5)
116-
$pb.PbList<$2.InternedString> get functionNames => $_getList(0);
157+
$pb.PbList<$3.InternedString> get functionNames => $_getList(3);
117158

118159
/// Frames of callstacks of a program.
119160
@$pb.TagNumber(6)
120-
$pb.PbList<$2.Frame> get frames => $_getList(1);
161+
$pb.PbList<$3.Frame> get frames => $_getList(4);
121162

122163
/// A callstack of a program.
123164
@$pb.TagNumber(7)
124-
$pb.PbList<$2.Callstack> get callstacks => $_getList(2);
165+
$pb.PbList<$3.Callstack> get callstacks => $_getList(5);
125166

126167
/// Paths to executable files.
127168
@$pb.TagNumber(17)
128-
$pb.PbList<$2.InternedString> get mappingPaths => $_getList(3);
169+
$pb.PbList<$3.InternedString> get mappingPaths => $_getList(6);
129170

130171
/// Executable files mapped into processes.
131172
@$pb.TagNumber(19)
132-
$pb.PbList<$2.Mapping> get mappings => $_getList(4);
173+
$pb.PbList<$3.Mapping> get mappings => $_getList(7);
174+
175+
/// Interned string values in the DebugAnnotation proto.
176+
@$pb.TagNumber(29)
177+
$pb.PbList<$3.InternedString> get debugAnnotationStringValues => $_getList(8);
133178
}
134179

135180
const _omitFieldNames = $core.bool.fromEnvironment('protobuf.omit_field_names');

‎pkg/vm_service_protos/lib/src/protos/perfetto/trace/interned_data/interned_data.pbjson.dart‎

Lines changed: 44 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,30 @@ import 'dart:typed_data' as $typed_data;
2525
const InternedData$json = {
2626
'1': 'InternedData',
2727
'2': [
28+
{
29+
'1': 'event_categories',
30+
'3': 1,
31+
'4': 3,
32+
'5': 11,
33+
'6': '.perfetto.protos.EventCategory',
34+
'10': 'eventCategories'
35+
},
36+
{
37+
'1': 'event_names',
38+
'3': 2,
39+
'4': 3,
40+
'5': 11,
41+
'6': '.perfetto.protos.EventName',
42+
'10': 'eventNames'
43+
},
44+
{
45+
'1': 'debug_annotation_names',
46+
'3': 3,
47+
'4': 3,
48+
'5': 11,
49+
'6': '.perfetto.protos.DebugAnnotationName',
50+
'10': 'debugAnnotationNames'
51+
},
2852
{
2953
'1': 'mapping_paths',
3054
'3': 17,
@@ -65,14 +89,28 @@ const InternedData$json = {
6589
'6': '.perfetto.protos.Callstack',
6690
'10': 'callstacks'
6791
},
92+
{
93+
'1': 'debug_annotation_string_values',
94+
'3': 29,
95+
'4': 3,
96+
'5': 11,
97+
'6': '.perfetto.protos.InternedString',
98+
'10': 'debugAnnotationStringValues'
99+
},
68100
],
69101
};
70102

71103
/// Descriptor for `InternedData`. Decode as a `google.protobuf.DescriptorProto`.
72104
final $typed_data.Uint8List internedDataDescriptor = $convert.base64Decode(
73-
'CgxJbnRlcm5lZERhdGESRAoNbWFwcGluZ19wYXRocxgRIAMoCzIfLnBlcmZldHRvLnByb3Rvcy'
74-
'5JbnRlcm5lZFN0cmluZ1IMbWFwcGluZ1BhdGhzEkYKDmZ1bmN0aW9uX25hbWVzGAUgAygLMh8u'
75-
'cGVyZmV0dG8ucHJvdG9zLkludGVybmVkU3RyaW5nUg1mdW5jdGlvbk5hbWVzEjQKCG1hcHBpbm'
76-
'dzGBMgAygLMhgucGVyZmV0dG8ucHJvdG9zLk1hcHBpbmdSCG1hcHBpbmdzEi4KBmZyYW1lcxgG'
77-
'IAMoCzIWLnBlcmZldHRvLnByb3Rvcy5GcmFtZVIGZnJhbWVzEjoKCmNhbGxzdGFja3MYByADKA'
78-
'syGi5wZXJmZXR0by5wcm90b3MuQ2FsbHN0YWNrUgpjYWxsc3RhY2tz');
105+
'CgxJbnRlcm5lZERhdGESSQoQZXZlbnRfY2F0ZWdvcmllcxgBIAMoCzIeLnBlcmZldHRvLnByb3'
106+
'Rvcy5FdmVudENhdGVnb3J5Ug9ldmVudENhdGVnb3JpZXMSOwoLZXZlbnRfbmFtZXMYAiADKAsy'
107+
'Gi5wZXJmZXR0by5wcm90b3MuRXZlbnROYW1lUgpldmVudE5hbWVzEloKFmRlYnVnX2Fubm90YX'
108+
'Rpb25fbmFtZXMYAyADKAsyJC5wZXJmZXR0by5wcm90b3MuRGVidWdBbm5vdGF0aW9uTmFtZVIU'
109+
'ZGVidWdBbm5vdGF0aW9uTmFtZXMSRAoNbWFwcGluZ19wYXRocxgRIAMoCzIfLnBlcmZldHRvLn'
110+
'Byb3Rvcy5JbnRlcm5lZFN0cmluZ1IMbWFwcGluZ1BhdGhzEkYKDmZ1bmN0aW9uX25hbWVzGAUg'
111+
'AygLMh8ucGVyZmV0dG8ucHJvdG9zLkludGVybmVkU3RyaW5nUg1mdW5jdGlvbk5hbWVzEjQKCG'
112+
'1hcHBpbmdzGBMgAygLMhgucGVyZmV0dG8ucHJvdG9zLk1hcHBpbmdSCG1hcHBpbmdzEi4KBmZy'
113+
'YW1lcxgGIAMoCzIWLnBlcmZldHRvLnByb3Rvcy5GcmFtZVIGZnJhbWVzEjoKCmNhbGxzdGFja3'
114+
'MYByADKAsyGi5wZXJmZXR0by5wcm90b3MuQ2FsbHN0YWNrUgpjYWxsc3RhY2tzEmQKHmRlYnVn'
115+
'X2Fubm90YXRpb25fc3RyaW5nX3ZhbHVlcxgdIAMoCzIfLnBlcmZldHRvLnByb3Rvcy5JbnRlcm'
116+
'5lZFN0cmluZ1IbZGVidWdBbm5vdGF0aW9uU3RyaW5nVmFsdWVz');

0 commit comments

Comments
 (0)