Skip to content

Commit ee2e45b

Browse files
mingmingl-llvmmingmingl-llvm
committed
text format
TODO: Support vtable profiles in binary SPGO format
1 parent 2664d70 commit ee2e45b

File tree

9 files changed

+163
-22
lines changed

9 files changed

+163
-22
lines changed

‎llvm/include/llvm/ProfileData/SampleProf.h‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -407,6 +407,7 @@ class SampleRecord {
407407
uint64_t getSamples() const { return NumSamples; }
408408
const CallTargetMap &getCallTargets() const { return CallTargets; }
409409
const TypeMap &getTypes() const { return TypeCounts; }
410+
TypeMap &getTypes() { return TypeCounts; }
410411
const SortedCallTargetSet getSortedCallTargets() const {
411412
return sortCallTargets(CallTargets);
412413
}
@@ -809,11 +810,22 @@ class FunctionSamples {
809810
Func, Num, Weight);
810811
}
811812

813+
sampleprof_error addTypeSamples(uint32_t LineOffset, uint32_t Discriminator,
814+
FunctionId Func, uint64_t Num,
815+
uint64_t Weight = 1) {
816+
return BodySamples[LineLocation(LineOffset, Discriminator)].addTypeCount(
817+
Func, Num, Weight);
818+
}
819+
812820
sampleprof_error addTypeSamples(const LineLocation &Loc, FunctionId Func,
813821
uint64_t Num, uint64_t Weight = 1) {
814822
return BodySamples[Loc].addTypeCount(Func, Num, Weight);
815823
}
816824

825+
TypeMap &getTypeSamples(const LineLocation &Loc) {
826+
return BodySamples[Loc].getTypes();
827+
}
828+
817829
sampleprof_error addSampleRecord(LineLocation Location,
818830
const SampleRecord &SampleRecord,
819831
uint64_t Weight = 1) {
@@ -1061,14 +1073,27 @@ class FunctionSamples {
10611073
const LineLocation &Loc = I.first;
10621074
const SampleRecord &Rec = I.second;
10631075
mergeSampleProfErrors(Result, BodySamples[Loc].merge(Rec, Weight));
1076+
// const auto &OtherTypeCountMap = Rec.getTypes();
1077+
// for (const auto &[Type, Count] : OtherTypeCountMap) {
1078+
// mergeSampleProfErrors(Result, addTypeSamples(Loc, Type, Count,
1079+
// Weight));
1080+
// }
10641081
}
1082+
10651083
for (const auto &I : Other.getCallsiteSamples()) {
10661084
const LineLocation &Loc = I.first;
10671085
FunctionSamplesMap &FSMap = functionSamplesAt(Loc);
10681086
for (const auto &Rec : I.second)
10691087
mergeSampleProfErrors(Result,
10701088
FSMap[Rec.first].merge(Rec.second, Weight));
10711089
}
1090+
for (const auto &[Loc, TypeCountMap] : Other.getCallsiteTypes()) {
1091+
TypeMap &TypeCounts = getTypeSamplesAt(Loc);
1092+
for (const auto &[Type, Count] : TypeCountMap) {
1093+
TypeCounts[Type] =
1094+
SaturatingMultiplyAdd(Count, Weight, TypeCounts[Type]);
1095+
}
1096+
}
10721097
return Result;
10731098
}
10741099

‎llvm/include/llvm/ProfileData/SampleProfWriter.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -431,7 +431,7 @@ class SampleProfileWriterExtBinary : public SampleProfileWriterExtBinaryBase {
431431
raw_ostream &OS) override;
432432

433433
void addTypeNames(const TypeMap &M) override {
434-
if (WriteVTableProf)
434+
if (!WriteVTableProf)
435435
return;
436436
// Add type name to TypeNameTable.
437437
for (const auto &[Type, Cnt] : M) {

‎llvm/lib/ProfileData/SampleProf.cpp‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,9 @@ sampleprof_error SampleRecord::merge(const SampleRecord &Other,
123123
for (const auto &I : Other.getCallTargets()) {
124124
mergeSampleProfErrors(Result, addCalledTarget(I.first, I.second, Weight));
125125
}
126+
for (const auto &[TypeName, Count] : Other.getTypes()) {
127+
mergeSampleProfErrors(Result, addTypeCount(TypeName, Count, Weight));
128+
}
126129
return Result;
127130
}
128131

@@ -138,6 +141,11 @@ void SampleRecord::print(raw_ostream &OS, unsigned Indent) const {
138141
for (const auto &I : getSortedCallTargets())
139142
OS << " " << I.first << ":" << I.second;
140143
}
144+
if (!TypeCounts.empty()) {
145+
OS << ", types:";
146+
for (const auto &I : TypeCounts)
147+
OS << " " << I.first << ":" << I.second;
148+
}
141149
OS << "\n";
142150
}
143151

@@ -179,11 +187,21 @@ void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const {
179187
SampleSorter<LineLocation, FunctionSamplesMap> SortedCallsiteSamples(
180188
CallsiteSamples);
181189
for (const auto &CS : SortedCallsiteSamples.get()) {
182-
for (const auto &FS : CS->second) {
190+
for (const auto &[FuncId, FuncSample] : CS->second) {
183191
OS.indent(Indent + 2);
184-
OS << CS->first << ": inlined callee: " << FS.second.getFunction()
192+
OS << CS->first << ": inlined callee: " << FuncSample.getFunction()
185193
<< ": ";
186-
FS.second.print(OS, Indent + 4);
194+
FuncSample.print(OS, Indent + 4);
195+
}
196+
const LineLocation &Loc = CS->first;
197+
auto TypeSamplesIter = VirtualCallsiteTypes.find(Loc);
198+
if (TypeSamplesIter != VirtualCallsiteTypes.end()) {
199+
OS.indent(Indent + 2);
200+
OS << Loc << ": vtables: ";
201+
for (const auto &TypeSample : TypeSamplesIter->second) {
202+
OS << TypeSample.first << ":" << TypeSample.second << " ";
203+
}
204+
OS << "\n";
187205
}
188206
}
189207
OS.indent(Indent);

‎llvm/lib/ProfileData/SampleProfReader.cpp‎

Lines changed: 72 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -197,8 +197,31 @@ enum class LineType {
197197
CallSiteProfile,
198198
BodyProfile,
199199
Metadata,
200+
CallTargetTypeProfile,
201+
CallSiteTypeProfile,
200202
};
201203

204+
static bool parseTypeCountMap(StringRef Input,
205+
DenseMap<StringRef, uint64_t> &TypeCountMap) {
206+
for (size_t Index = Input.find_first_not_of(' '); Index != StringRef::npos;) {
207+
size_t n1 = Input.find(':', Index);
208+
if (n1 == StringRef::npos)
209+
return false; // No colon found, invalid format.
210+
StringRef TypeName = Input.substr(Index, n1 - Index);
211+
// n2 is the start index of count.
212+
size_t n2 = n1 + 1;
213+
// n3 is the start index after the 'target:count' pair.
214+
size_t n3 = Input.find_first_of(' ', n2);
215+
uint64_t Count;
216+
if (Input.substr(n2, n3 - n2).getAsInteger(10, Count))
217+
return false; // Invalid count.
218+
TypeCountMap[TypeName] = Count;
219+
Index = (n3 == StringRef::npos) ? StringRef::npos
220+
: Input.find_first_not_of(' ', n3);
221+
}
222+
return true;
223+
}
224+
202225
/// Parse \p Input as line sample.
203226
///
204227
/// \param Input input line.
@@ -215,6 +238,7 @@ static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,
215238
uint64_t &NumSamples, uint32_t &LineOffset,
216239
uint32_t &Discriminator, StringRef &CalleeName,
217240
DenseMap<StringRef, uint64_t> &TargetCountMap,
241+
DenseMap<StringRef, uint64_t> &TypeCountMap,
218242
uint64_t &FunctionHash, uint32_t &Attributes,
219243
bool &IsFlat) {
220244
for (Depth = 0; Input[Depth] == ' '; Depth++)
@@ -289,6 +313,7 @@ static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,
289313
n4 = AfterColon.find_first_of(' ');
290314
n4 = (n4 != StringRef::npos) ? n3 + n4 + 1 : Rest.size();
291315
StringRef WordAfterColon = Rest.substr(n3 + 1, n4 - n3 - 1);
316+
// Break the loop if parsing integer succeeded.
292317
if (!WordAfterColon.getAsInteger(10, count))
293318
break;
294319

@@ -306,6 +331,16 @@ static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,
306331
// Change n3 to the next blank space after colon + integer pair.
307332
n3 = n4;
308333
}
334+
} else if (Rest.ends_with("// CallTargetVtables")) {
335+
LineTy = LineType::CallTargetTypeProfile;
336+
return parseTypeCountMap(
337+
Rest.substr(0, Rest.size() - strlen("// CallTargetVtables")),
338+
TypeCountMap);
339+
} else if (Rest.ends_with("// CallSiteVtables")) {
340+
LineTy = LineType::CallSiteTypeProfile;
341+
return parseTypeCountMap(
342+
Rest.substr(0, Rest.size() - strlen("// CallSiteVtables")),
343+
TypeCountMap);
309344
} else {
310345
LineTy = LineType::CallSiteProfile;
311346
size_t n3 = Rest.find_last_of(':');
@@ -374,14 +409,15 @@ std::error_code SampleProfileReaderText::readImpl() {
374409
uint64_t NumSamples;
375410
StringRef FName;
376411
DenseMap<StringRef, uint64_t> TargetCountMap;
412+
DenseMap<StringRef, uint64_t> TypeCountMap;
377413
uint32_t Depth, LineOffset, Discriminator;
378414
LineType LineTy;
379415
uint64_t FunctionHash = 0;
380416
uint32_t Attributes = 0;
381417
bool IsFlat = false;
382418
if (!ParseLine(*LineIt, LineTy, Depth, NumSamples, LineOffset,
383-
Discriminator, FName, TargetCountMap, FunctionHash,
384-
Attributes, IsFlat)) {
419+
Discriminator, FName, TargetCountMap, TypeCountMap,
420+
FunctionHash, Attributes, IsFlat)) {
385421
reportError(LineIt.line_number(),
386422
"Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +
387423
*LineIt);
@@ -410,6 +446,29 @@ std::error_code SampleProfileReaderText::readImpl() {
410446
DepthMetadata = 0;
411447
break;
412448
}
449+
450+
case LineType::CallSiteTypeProfile: {
451+
TypeMap &Map = InlineStack.back()->getTypeSamplesAt(
452+
LineLocation(LineOffset, Discriminator));
453+
for (const auto [Type, Count] : TypeCountMap)
454+
Map[FunctionId(Type)] += Count;
455+
break;
456+
}
457+
458+
case LineType::CallTargetTypeProfile: {
459+
while (InlineStack.size() > Depth) {
460+
InlineStack.pop_back();
461+
}
462+
FunctionSamples &FProfile = *InlineStack.back();
463+
for (const auto &name_count : TypeCountMap) {
464+
mergeSampleProfErrors(
465+
Result, FProfile.addTypeSamples(LineOffset, Discriminator,
466+
FunctionId(name_count.first),
467+
name_count.second));
468+
}
469+
break;
470+
}
471+
413472
case LineType::BodyProfile: {
414473
while (InlineStack.size() > Depth) {
415474
InlineStack.pop_back();
@@ -608,6 +667,7 @@ std::error_code SampleProfileReaderExtBinary::readTypeMap(TypeMap &M) {
608667
if (std::error_code EC = VTableSamples.getError())
609668
return EC;
610669

670+
errs() << "readTypeMap\t" << *VTableType << "\t" << *VTableSamples << "\n";
611671
M.insert(std::make_pair(*VTableType, *VTableSamples));
612672
}
613673
return sampleprof_error::success;
@@ -619,7 +679,7 @@ SampleProfileReaderExtBinary::readVTableProf(const LineLocation &Loc,
619679
if (!ReadVTableProf)
620680
return sampleprof_error::success;
621681

622-
return readTypeMap(FProfile.getTypeSamplesAt(Loc));
682+
return readTypeMap(FProfile.getTypeSamples(Loc));
623683
}
624684

625685
std::error_code SampleProfileReaderExtBinary::readCallsiteVTableProf(
@@ -648,8 +708,8 @@ std::error_code SampleProfileReaderExtBinary::readCallsiteVTableProf(
648708
return std::error_code();
649709
}
650710

651-
if (std::error_code EC = readVTableProf(
652-
LineLocation(*LineOffset, DiscriminatorVal), FProfile))
711+
if (std::error_code EC = readTypeMap(FProfile.getTypeSamplesAt(
712+
LineLocation(*LineOffset, DiscriminatorVal))))
653713
return EC;
654714
}
655715
return sampleprof_error::success;
@@ -740,7 +800,10 @@ SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {
740800
return EC;
741801
}
742802

743-
return readCallsiteVTableProf(FProfile);
803+
std::error_code EC = readCallsiteVTableProf(FProfile);
804+
errs() << "readFunctionSample\t";
805+
FProfile.print(errs(), 2);
806+
return EC;
744807
}
745808

746809
std::error_code
@@ -802,8 +865,10 @@ std::error_code SampleProfileReaderExtBinaryBase::readOneSection(
802865
FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined = true;
803866
if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFSDiscriminator))
804867
FunctionSamples::ProfileIsFS = ProfileIsFS = true;
805-
if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagHasVTableTypeProf))
868+
if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagHasVTableTypeProf)) {
869+
errs() << "SampleProfileReaderExtBinaryBase::readVTableProf\n";
806870
ReadVTableProf = true;
871+
}
807872
break;
808873
case SecNameTable: {
809874
bool FixedLengthMD5 =

‎llvm/lib/ProfileData/SampleProfWriter.cpp‎

Lines changed: 30 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -489,7 +489,10 @@ SampleProfileWriterExtBinary::SampleProfileWriterExtBinary(
489489
std::unique_ptr<raw_ostream> &OS)
490490
: SampleProfileWriterExtBinaryBase(OS) {
491491
// Initialize the section header layout.
492+
492493
WriteVTableProf = ExtBinaryWriteVTableTypeProf;
494+
495+
errs() << "writeVTableProf value: " << WriteVTableProf << "\n";
493496
}
494497

495498
std::error_code SampleProfileWriterExtBinary::writeDefaultLayout(
@@ -604,15 +607,25 @@ std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
604607
for (const auto &J : Sample.getSortedCallTargets())
605608
OS << " " << J.first << ":" << J.second;
606609
OS << "\n";
610+
611+
if (!Sample.getTypes().empty()) {
612+
OS.indent(Indent + 1);
613+
Loc.print(OS);
614+
OS << ": ";
615+
for (const auto &Type : Sample.getTypes()) {
616+
OS << Type.first << ":" << Type.second << " ";
617+
}
618+
OS << " // CallTargetVtables\n";
619+
}
607620
LineCount++;
608621
}
609622

610623
SampleSorter<LineLocation, FunctionSamplesMap> SortedCallsiteSamples(
611624
S.getCallsiteSamples());
612625
Indent += 1;
613-
for (const auto &I : SortedCallsiteSamples.get())
626+
for (const auto &I : SortedCallsiteSamples.get()) {
627+
LineLocation Loc = I->first;
614628
for (const auto &FS : I->second) {
615-
LineLocation Loc = I->first;
616629
const FunctionSamples &CalleeSamples = FS.second;
617630
OS.indent(Indent);
618631
if (Loc.Discriminator == 0)
@@ -622,6 +635,19 @@ std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
622635
if (std::error_code EC = writeSample(CalleeSamples))
623636
return EC;
624637
}
638+
639+
if (const TypeMap *Map = S.findTypeSamplesAt(Loc); Map && !Map->empty()) {
640+
OS.indent(Indent);
641+
Loc.print(OS);
642+
OS << ": ";
643+
for (const auto &Type : *Map) {
644+
OS << Type.first << ":" << Type.second << " ";
645+
}
646+
OS << " // CallSiteVtables\n";
647+
LineCount++;
648+
}
649+
}
650+
625651
Indent -= 1;
626652

627653
if (FunctionSamples::ProfileIsProbeBased) {
@@ -828,6 +854,8 @@ std::error_code SampleProfileWriterExtBinary::writeTypeMap(const TypeMap &Map,
828854
raw_ostream &OS) {
829855
encodeULEB128(Map.size(), OS);
830856
for (const auto &[TypeName, TypeSamples] : Map) {
857+
errs() << "TypeName: " << TypeName << "\t" << "TypeSamples: " << TypeSamples
858+
<< "\n";
831859
if (std::error_code EC = writeNameIdx(TypeName))
832860
return EC;
833861
encodeULEB128(TypeSamples, OS);
@@ -840,8 +868,6 @@ std::error_code SampleProfileWriterExtBinary::writeSampleRecordVTableProf(
840868
if (!WriteVTableProf)
841869
return sampleprof_error::success;
842870

843-
// TODO: Unify this with SampleProfileWriterBinary::writeBody with a
844-
// pre-commit refactor.
845871
return writeTypeMap(Record.getTypes(), OS);
846872
}
847873

@@ -851,7 +877,6 @@ std::error_code SampleProfileWriterExtBinary::writeCallsiteType(
851877
return sampleprof_error::success;
852878

853879
const CallsiteTypeMap &CallsiteTypeMap = FunctionSample.getCallsiteTypes();
854-
855880
encodeULEB128(CallsiteTypeMap.size(), OS);
856881
for (const auto &[Loc, TypeMap] : CallsiteTypeMap) {
857882
encodeULEB128(Loc.LineOffset, OS);

‎llvm/test/tools/llvm-profdata/Inputs/profile-symbol-list.expected‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ Samples collected in the function's body {
66
5.1: 2150
77
6: 4160
88
7: 1068
9-
9: 4128, calls: _Z3bari:2942 _Z3fooi:1262
9+
9: 4128, calls: _Z3bari:2942 _Z3fooi:1262, types: vtable_bar:2942 vtable_foo:1260
1010
}
1111
Samples collected in inlined callsites {
1212
10: inlined callee: inline1: 2000, 0, 1 sampled lines
@@ -19,6 +19,7 @@ Samples collected in inlined callsites {
1919
1: 4000
2020
}
2121
No inlined callsites in this function
22+
10: vtables: inline1_vtable:2000 inline2_vtable:4000
2223
}
2324
Function: _Z3bari: 40602, 2874, 1 sampled lines
2425
Samples collected in the function's body {

‎llvm/test/tools/llvm-profdata/Inputs/sample-profile.proftext‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@ main:184019:0
66
6: 2080
77
7: 534
88
9: 2064 _Z3bari:1471 _Z3fooi:631
9+
9: vtable_bar:1471 vtable_foo:630 // CallTargetVtables
910
10: inline1:1000
1011
1: 1000
1112
10: inline2:2000
1213
1: 2000
14+
10: inline1_vtable:1000 inline2_vtable:2000 // CallSiteVtables
1315
_Z3bari:20301:1437
1416
1: 1437
1517
_Z3fooi:7711:610

0 commit comments

Comments
 (0)