Skip to content

Commit 3e01bf6

Browse files
aaupovaaupov
committed
[𝘀𝗽𝗿] initial version
Created using spr 1.3.4
2 parents 88738a7 + dc30825 commit 3e01bf6

File tree

12 files changed

+56
-72
lines changed

12 files changed

+56
-72
lines changed

‎bolt/include/bolt/Core/BinaryFunction.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,10 @@ class BinaryFunction {
388388
/// The profile data for the number of times the function was executed.
389389
uint64_t ExecutionCount{COUNT_NO_PROFILE};
390390

391+
/// Profile data for the number of times this function was entered from
392+
/// external code (DSO, JIT, etc).
393+
uint64_t ExternEntryCount{0};
394+
391395
/// Profile match ratio.
392396
float ProfileMatchRatio{0.0f};
393397

@@ -1877,6 +1881,10 @@ class BinaryFunction {
18771881
return *this;
18781882
}
18791883

1884+
/// Set the profile data for the number of times the function was entered from
1885+
/// external code (DSO/JIT).
1886+
void setExternEntryCount(uint64_t Count) { ExternEntryCount = Count; }
1887+
18801888
/// Adjust execution count for the function by a given \p Count. The value
18811889
/// \p Count will be subtracted from the current function count.
18821890
///
@@ -1904,6 +1912,10 @@ class BinaryFunction {
19041912
/// Return COUNT_NO_PROFILE if there's no profile info.
19051913
uint64_t getExecutionCount() const { return ExecutionCount; }
19061914

1915+
/// Return the profile information about the number of times the function was
1916+
/// entered from external code (DSO/JIT).
1917+
uint64_t getExternEntryCount() const { return ExternEntryCount; }
1918+
19071919
/// Return the raw profile information about the number of branch
19081920
/// executions corresponding to this function.
19091921
uint64_t getRawSampleCount() const { return RawSampleCount; }

‎bolt/include/bolt/Profile/DataAggregator.h‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,13 @@ class DataAggregator : public DataReader {
7878
static bool checkPerfDataMagic(StringRef FileName);
7979

8080
private:
81+
struct LBREntry {
82+
uint64_t From;
83+
uint64_t To;
84+
bool Mispred;
85+
};
86+
friend raw_ostream &operator<<(raw_ostream &OS, const LBREntry &);
87+
8188
struct PerfBranchSample {
8289
SmallVector<LBREntry, 32> LBR;
8390
};
@@ -476,7 +483,6 @@ class DataAggregator : public DataReader {
476483

477484
/// Debugging dump methods
478485
void dump() const;
479-
void dump(const LBREntry &LBR) const;
480486
void dump(const PerfBranchSample &Sample) const;
481487
void dump(const PerfMemSample &Sample) const;
482488

@@ -504,6 +510,12 @@ class DataAggregator : public DataReader {
504510

505511
friend class YAMLProfileWriter;
506512
};
513+
514+
inline raw_ostream &operator<<(raw_ostream &OS,
515+
const DataAggregator::LBREntry &L) {
516+
OS << formatv("{0:x} -> {1:x}/{2}", L.From, L.To, L.Mispred ? 'M' : 'P');
517+
return OS;
518+
}
507519
} // namespace bolt
508520
} // namespace llvm
509521

‎bolt/include/bolt/Profile/DataReader.h‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -32,18 +32,6 @@ namespace bolt {
3232

3333
class BinaryFunction;
3434

35-
struct LBREntry {
36-
uint64_t From;
37-
uint64_t To;
38-
bool Mispred;
39-
};
40-
41-
inline raw_ostream &operator<<(raw_ostream &OS, const LBREntry &LBR) {
42-
OS << "0x" << Twine::utohexstr(LBR.From) << " -> 0x"
43-
<< Twine::utohexstr(LBR.To);
44-
return OS;
45-
}
46-
4735
struct Location {
4836
bool IsSymbol;
4937
StringRef Name;
@@ -109,6 +97,9 @@ struct FuncBranchData {
10997
/// Total execution count for the function.
11098
int64_t ExecutionCount{0};
11199

100+
/// Total entry count from external code for the function.
101+
uint64_t ExternEntryCount{0};
102+
112103
/// Indicate if the data was used.
113104
bool Used{false};
114105

‎bolt/include/bolt/Profile/ProfileYAMLMapping.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,7 @@ struct BinaryFunctionProfile {
206206
uint32_t Id{0};
207207
llvm::yaml::Hex64 Hash{0};
208208
uint64_t ExecCount{0};
209+
uint64_t ExternEntryCount{0};
209210
std::vector<BinaryBasicBlockProfile> Blocks;
210211
std::vector<InlineTreeNode> InlineTree;
211212
bool Used{false};
@@ -218,6 +219,7 @@ template <> struct MappingTraits<bolt::BinaryFunctionProfile> {
218219
YamlIO.mapRequired("fid", BFP.Id);
219220
YamlIO.mapRequired("hash", BFP.Hash);
220221
YamlIO.mapRequired("exec", BFP.ExecCount);
222+
YamlIO.mapOptional("extern", BFP.ExternEntryCount, 0);
221223
YamlIO.mapRequired("nblocks", BFP.NumBasicBlocks);
222224
YamlIO.mapOptional("blocks", BFP.Blocks,
223225
std::vector<bolt::BinaryBasicBlockProfile>());

‎bolt/lib/Core/BinaryFunction.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -471,6 +471,8 @@ void BinaryFunction::print(raw_ostream &OS, std::string Annotation) {
471471
OS << "\n Sample Count: " << RawSampleCount;
472472
OS << "\n Profile Acc : " << format("%.1f%%", ProfileMatchRatio * 100.0f);
473473
}
474+
if (ExternEntryCount)
475+
OS << "\n Extern Entry Count: " << ExternEntryCount;
474476

475477
if (opts::PrintDynoStats && !getLayout().block_empty()) {
476478
OS << '\n';

‎bolt/lib/Passes/ProfileQualityStats.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -532,6 +532,9 @@ void computeFlowMappings(const BinaryContext &BC, FlowInfo &TotalFlowMap) {
532532
std::vector<uint64_t> &MaxCountMap = TotalMaxCountMaps[FunctionNum];
533533
std::vector<uint64_t> &MinCountMap = TotalMinCountMaps[FunctionNum];
534534

535+
// Record external entry count into CallGraphIncomingFlows
536+
CallGraphIncomingFlows[FunctionNum] += Function->getExternEntryCount();
537+
535538
// Update MaxCountMap, MinCountMap, and CallGraphIncomingFlows
536539
auto recordCall = [&](const BinaryBasicBlock *SourceBB,
537540
const MCSymbol *DestSymbol, uint64_t Count,

‎bolt/lib/Profile/BoltAddressTranslation.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -546,7 +546,7 @@ BoltAddressTranslation::getFallthroughsInTrace(uint64_t FuncAddress,
546546
return Res;
547547

548548
for (auto Iter = FromIter; Iter != ToIter;) {
549-
const uint32_t Src = Iter->first;
549+
const uint32_t Src = Iter->second >> 1;
550550
if (Iter->second & BRANCHENTRY) {
551551
++Iter;
552552
continue;
@@ -557,7 +557,7 @@ BoltAddressTranslation::getFallthroughsInTrace(uint64_t FuncAddress,
557557
++Iter;
558558
if (Iter->second & BRANCHENTRY)
559559
break;
560-
Res.emplace_back(Src, Iter->first);
560+
Res.emplace_back(Src, Iter->second >> 1);
561561
}
562562

563563
return Res;

‎bolt/lib/Profile/DataAggregator.cpp‎

Lines changed: 9 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -827,13 +827,8 @@ bool DataAggregator::doTrace(const LBREntry &First, const LBREntry &Second,
827827
<< FromFunc->getPrintName() << ":"
828828
<< Twine::utohexstr(First.To) << " to "
829829
<< Twine::utohexstr(Second.From) << ".\n");
830-
for (auto [From, To] : *FTs) {
831-
if (BAT) {
832-
From = BAT->translate(FromFunc->getAddress(), From, /*IsBranchSrc=*/true);
833-
To = BAT->translate(FromFunc->getAddress(), To, /*IsBranchSrc=*/false);
834-
}
830+
for (auto [From, To] : *FTs)
835831
doIntraBranch(*ParentFunc, From, To, Count, false);
836-
}
837832

838833
return true;
839834
}
@@ -972,7 +967,7 @@ bool DataAggregator::recordExit(BinaryFunction &BF, uint64_t From, bool Mispred,
972967
return true;
973968
}
974969

975-
ErrorOr<LBREntry> DataAggregator::parseLBREntry() {
970+
ErrorOr<DataAggregator::LBREntry> DataAggregator::parseLBREntry() {
976971
LBREntry Res;
977972
ErrorOr<StringRef> FromStrRes = parseString('/');
978973
if (std::error_code EC = FromStrRes.getError())
@@ -1430,54 +1425,16 @@ void DataAggregator::parseLBRSample(const PerfBranchSample &Sample,
14301425
const uint64_t TraceTo = NextLBR->From;
14311426
const BinaryFunction *TraceBF =
14321427
getBinaryFunctionContainingAddress(TraceFrom);
1433-
if (opts::HeatmapMode == opts::HeatmapModeKind::HM_Exclusive) {
1434-
FTInfo &Info = FallthroughLBRs[Trace(TraceFrom, TraceTo)];
1428+
FTInfo &Info = FallthroughLBRs[Trace(TraceFrom, TraceTo)];
1429+
if (TraceBF->containsAddress(LBR.From))
14351430
++Info.InternCount;
1436-
} else if (TraceBF && TraceBF->containsAddress(TraceTo)) {
1437-
FTInfo &Info = FallthroughLBRs[Trace(TraceFrom, TraceTo)];
1438-
if (TraceBF->containsAddress(LBR.From))
1439-
++Info.InternCount;
1440-
else
1441-
++Info.ExternCount;
1442-
} else {
1443-
const BinaryFunction *ToFunc =
1444-
getBinaryFunctionContainingAddress(TraceTo);
1445-
if (TraceBF && ToFunc) {
1446-
LLVM_DEBUG({
1447-
dbgs() << "Invalid trace starting in " << TraceBF->getPrintName()
1448-
<< formatv(" @ {0:x}", TraceFrom - TraceBF->getAddress())
1449-
<< formatv(" and ending @ {0:x}\n", TraceTo);
1450-
});
1451-
++NumInvalidTraces;
1452-
} else {
1453-
LLVM_DEBUG({
1454-
dbgs() << "Out of range trace starting in "
1455-
<< (TraceBF ? TraceBF->getPrintName() : "None")
1456-
<< formatv(" @ {0:x}",
1457-
TraceFrom - (TraceBF ? TraceBF->getAddress() : 0))
1458-
<< " and ending in "
1459-
<< (ToFunc ? ToFunc->getPrintName() : "None")
1460-
<< formatv(" @ {0:x}\n",
1461-
TraceTo - (ToFunc ? ToFunc->getAddress() : 0));
1462-
});
1463-
++NumLongRangeTraces;
1464-
}
1465-
}
1431+
else
1432+
++Info.ExternCount;
14661433
++NumTraces;
14671434
}
14681435
NextLBR = &LBR;
14691436

1470-
// Record branches outside binary functions for heatmap.
1471-
if (opts::HeatmapMode == opts::HeatmapModeKind::HM_Exclusive) {
1472-
TakenBranchInfo &Info = BranchLBRs[Trace(LBR.From, LBR.To)];
1473-
++Info.TakenCount;
1474-
continue;
1475-
}
1476-
uint64_t From = getBinaryFunctionContainingAddress(LBR.From) ? LBR.From : 0;
1477-
uint64_t To = getBinaryFunctionContainingAddress(LBR.To) ? LBR.To : 0;
1478-
if (!From && !To)
1479-
continue;
1480-
TakenBranchInfo &Info = BranchLBRs[Trace(From, To)];
1437+
TakenBranchInfo &Info = BranchLBRs[Trace(LBR.From, LBR.To)];
14811438
++Info.TakenCount;
14821439
Info.MispredCount += LBR.Mispred;
14831440
}
@@ -2289,6 +2246,7 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
22892246
YamlBF.Id = BF->getFunctionNumber();
22902247
YamlBF.Hash = BAT->getBFHash(FuncAddress);
22912248
YamlBF.ExecCount = BF->getKnownExecutionCount();
2249+
YamlBF.ExternEntryCount = BF->getExternEntryCount();
22922250
YamlBF.NumBasicBlocks = BAT->getNumBasicBlocks(FuncAddress);
22932251
const BoltAddressTranslation::BBHashMapTy &BlockMap =
22942252
BAT->getBBHashMap(FuncAddress);
@@ -2398,16 +2356,10 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
23982356

23992357
void DataAggregator::dump() const { DataReader::dump(); }
24002358

2401-
void DataAggregator::dump(const LBREntry &LBR) const {
2402-
Diag << "From: " << Twine::utohexstr(LBR.From)
2403-
<< " To: " << Twine::utohexstr(LBR.To) << " Mispred? " << LBR.Mispred
2404-
<< "\n";
2405-
}
2406-
24072359
void DataAggregator::dump(const PerfBranchSample &Sample) const {
24082360
Diag << "Sample LBR entries: " << Sample.LBR.size() << "\n";
24092361
for (const LBREntry &LBR : Sample.LBR)
2410-
dump(LBR);
2362+
Diag << LBR << '\n';
24112363
}
24122364

24132365
void DataAggregator::dump(const PerfMemSample &Sample) const {

‎bolt/lib/Profile/DataReader.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ void FuncBranchData::appendFrom(const FuncBranchData &FBD, uint64_t Offset) {
8585
}
8686
llvm::stable_sort(Data);
8787
ExecutionCount += FBD.ExecutionCount;
88+
ExternEntryCount += FBD.ExternEntryCount;
8889
for (auto I = FBD.EntryData.begin(), E = FBD.EntryData.end(); I != E; ++I) {
8990
assert(I->To.Name == FBD.Name);
9091
auto NewElmt = EntryData.insert(EntryData.end(), *I);
@@ -269,6 +270,7 @@ Error DataReader::preprocessProfile(BinaryContext &BC) {
269270
if (FuncBranchData *FuncData = getBranchDataForNames(Function.getNames())) {
270271
setBranchData(Function, FuncData);
271272
Function.ExecutionCount = FuncData->ExecutionCount;
273+
Function.ExternEntryCount = FuncData->ExternEntryCount;
272274
FuncData->Used = true;
273275
}
274276
}
@@ -419,6 +421,7 @@ void DataReader::matchProfileData(BinaryFunction &BF) {
419421
if (fetchProfileForOtherEntryPoints(BF)) {
420422
BF.ProfileMatchRatio = evaluateProfileData(BF, *FBD);
421423
BF.ExecutionCount = FBD->ExecutionCount;
424+
BF.ExternEntryCount = FBD->ExternEntryCount;
422425
BF.RawSampleCount = FBD->getNumExecutedBranches();
423426
}
424427
return;
@@ -449,6 +452,7 @@ void DataReader::matchProfileData(BinaryFunction &BF) {
449452
setBranchData(BF, NewBranchData);
450453
NewBranchData->Used = true;
451454
BF.ExecutionCount = NewBranchData->ExecutionCount;
455+
BF.ExternEntryCount = NewBranchData->ExternEntryCount;
452456
BF.ProfileMatchRatio = 1.0f;
453457
break;
454458
}
@@ -1190,6 +1194,8 @@ std::error_code DataReader::parse() {
11901194
if (BI.To.IsSymbol && BI.To.Offset == 0) {
11911195
I = GetOrCreateFuncEntry(BI.To.Name);
11921196
I->second.ExecutionCount += BI.Branches;
1197+
if (!BI.From.IsSymbol)
1198+
I->second.ExternEntryCount += BI.Branches;
11931199
}
11941200
}
11951201

‎bolt/lib/Profile/YAMLProfileReader.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,7 @@ bool YAMLProfileReader::parseFunctionProfile(
176176
uint64_t FunctionExecutionCount = 0;
177177

178178
BF.setExecutionCount(YamlBF.ExecCount);
179+
BF.setExternEntryCount(YamlBF.ExternEntryCount);
179180

180181
uint64_t FuncRawBranchCount = 0;
181182
for (const yaml::bolt::BinaryBasicBlockProfile &YamlBB : YamlBF.Blocks)

0 commit comments

Comments
 (0)