[StaticDataLayout] Reconcile data hotness based on PGO counters and data access profiles #163325
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,10 +1,14 @@ | ||
| #include "llvm/Analysis/StaticDataProfileInfo.h" | ||
| #include "llvm/Analysis/ProfileSummaryInfo.h" | ||
| #include "llvm/IR/Constant.h" | ||
| #include "llvm/IR/Constants.h" | ||
| #include "llvm/IR/GlobalVariable.h" | ||
| #include "llvm/IR/Module.h" | ||
| #include "llvm/InitializePasses.h" | ||
| #include "llvm/ProfileData/InstrProf.h" | ||
|
|
||
| #define DEBUG_TYPE "static-data-profile-info" | ||
|
|
||
| using namespace llvm; | ||
|
|
||
| namespace llvm { | ||
|
|
@@ -46,6 +50,12 @@ bool IsAnnotationOK(const GlobalVariable &GV) { | |
| } // namespace memprof | ||
| } // namespace llvm | ||
|
|
||
| #ifndef NDEBUG | ||
| static StringRef debugPrintSectionPrefix(StringRef Prefix) { | ||
|
||
| return Prefix.empty() ? "<empty>" : Prefix; | ||
| } | ||
| #endif | ||
|
|
||
| void StaticDataProfileInfo::addConstantProfileCount( | ||
| const Constant *C, std::optional<uint64_t> Count) { | ||
| if (!Count) { | ||
|
|
@@ -60,6 +70,49 @@ void StaticDataProfileInfo::addConstantProfileCount( | |
| OriginalCount = getInstrMaxCountValue(); | ||
| } | ||
|
|
||
| StaticDataProfileInfo::StaticDataHotness | ||
| StaticDataProfileInfo::getSectionHotnessUsingProfileCount( | ||
| const Constant *C, const ProfileSummaryInfo *PSI, uint64_t Count) const { | ||
| // The accummulated counter shows the constant is hot. Return 'hot' whether | ||
| // this variable is seen by unprofiled functions or not. | ||
| if (PSI->isHotCount(Count)) | ||
| return StaticDataHotness::Hot; | ||
| // The constant is not hot, and seen by unprofiled functions. We don't want to | ||
| // assign it to unlikely sections, even if the counter says 'cold'. So return | ||
| // an empty prefix before checking whether the counter is cold. | ||
| if (ConstantWithoutCounts.count(C)) | ||
| return StaticDataHotness::LukewarmOrUnknown; | ||
| // The accummulated counter shows the constant is cold. Return 'unlikely'. | ||
| if (PSI->isColdCount(Count)) | ||
| return StaticDataHotness::Cold; | ||
|
|
||
| return StaticDataHotness::LukewarmOrUnknown; | ||
| } | ||
|
|
||
| StaticDataProfileInfo::StaticDataHotness | ||
| StaticDataProfileInfo::getSectionHotnessUsingDAP( | ||
| std::optional<StringRef> MaybeSectionPrefix) const { | ||
| if (!MaybeSectionPrefix) | ||
| return StaticDataProfileInfo::StaticDataHotness::LukewarmOrUnknown; | ||
| StringRef Prefix = *MaybeSectionPrefix; | ||
| assert((Prefix == "hot" || Prefix == "unlikely") && | ||
| "Expect section_prefix to be one of hot or unlikely"); | ||
| return Prefix == "hot" ? StaticDataProfileInfo::StaticDataHotness::Hot | ||
| : StaticDataProfileInfo::StaticDataHotness::Cold; | ||
| } | ||
|
|
||
| StringRef StaticDataProfileInfo::hotnessToStr( | ||
| StaticDataProfileInfo::StaticDataHotness Hotness) const { | ||
| switch (Hotness) { | ||
| case StaticDataProfileInfo::StaticDataHotness::Cold: | ||
| return "unlikely"; | ||
| case StaticDataProfileInfo::StaticDataHotness::Hot: | ||
| return "hot"; | ||
| default: | ||
| return ""; | ||
| } | ||
| } | ||
|
|
||
| std::optional<uint64_t> | ||
| StaticDataProfileInfo::getConstantProfileCount(const Constant *C) const { | ||
| auto I = ConstantProfileCounts.find(C); | ||
|
|
@@ -70,27 +123,49 @@ StaticDataProfileInfo::getConstantProfileCount(const Constant *C) const { | |
|
|
||
| StringRef StaticDataProfileInfo::getConstantSectionPrefix( | ||
| const Constant *C, const ProfileSummaryInfo *PSI) const { | ||
| std::optional<uint64_t> Count = getConstantProfileCount(C); | ||
|
|
||
| if (EnableDataAccessProf) { | ||
| // Module flag `HasDataAccessProf` is 1 -> empty section prefix means | ||
| // unknown hotness except for string literals. | ||
| if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(C); | ||
| GV && llvm::memprof::IsAnnotationOK(*GV) && | ||
| !GV->getName().starts_with(".str")) { | ||
| auto HotnessFromDAP = getSectionHotnessUsingDAP(GV->getSectionPrefix()); | ||
|
|
||
| if (!Count) { | ||
| StringRef Prefix = hotnessToStr(HotnessFromDAP); | ||
| LLVM_DEBUG(dbgs() << GV->getName() << "has section prefix " | ||
| << debugPrintSectionPrefix(Prefix) | ||
| << ", solely from data access profiles\n"); | ||
| return Prefix; | ||
| } | ||
|
|
||
| // Both DAP and PGO counters are available. Use the hotter one. | ||
| auto HotnessFromPGO = getSectionHotnessUsingProfileCount(C, PSI, *Count); | ||
| StringRef Prefix = hotnessToStr(std::max(HotnessFromDAP, HotnessFromPGO)); | ||
|
||
| LLVM_DEBUG(dbgs() << GV->getName() << " has section prefix " | ||
| << debugPrintSectionPrefix(Prefix) | ||
| << ", the max from DAP as " | ||
| << debugPrintSectionPrefix(hotnessToStr(HotnessFromDAP)) | ||
| << " and PGO counters as " | ||
| << debugPrintSectionPrefix(hotnessToStr(HotnessFromPGO)) | ||
| << "\n"); | ||
| return Prefix; | ||
| } | ||
| } | ||
| if (!Count) | ||
| return ""; | ||
|
|
||
| return hotnessToStr(getSectionHotnessUsingProfileCount(C, PSI, *Count)); | ||
| } | ||
|
|
||
| bool StaticDataProfileInfoWrapperPass::doInitialization(Module &M) { | ||
| bool EnableDataAccessProf = false; | ||
| if (auto *MD = mdconst::extract_or_null<ConstantInt>( | ||
| M.getModuleFlag("EnableDataAccessProf"))) | ||
| EnableDataAccessProf = MD->getZExtValue(); | ||
| Info.reset(new StaticDataProfileInfo(EnableDataAccessProf)); | ||
| return false; | ||
| } | ||
|
|
||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I see the DAP suffix used here and also in a few variable names. I wonder how we can make it easier for the reader to know that it's an abbreviation for
DataAccessProfile?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
done by extending 'DAP' to
DataAccessProfileorDataAccessProf. I'll think about where to mention 'DAP' to LLVM docs so the abbreviation name are widely known (like PGO/LTO, etc).I'll revise the test name
llvm/test/CodeGen/X86/global-variable-partition-with-dap.llin a follow-up patch.