Skip to content

Commit ed74396

Browse files
mingmingl-llvmmingmingl-llvm
committed
reconcile hotness
1 parent 9a177c2 commit ed74396

File tree

3 files changed

+163
-16
lines changed

3 files changed

+163
-16
lines changed

‎llvm/include/llvm/Analysis/StaticDataProfileInfo.h‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,18 @@ class StaticDataProfileInfo {
5858
LLVM_ABI StaticDataHotness getSectionHotnessUsingProfileCount(
5959
const Constant *C, const ProfileSummaryInfo *PSI, uint64_t Count) const;
6060

61+
/// Return the hotness based on section prefix \p SectionPrefix.
62+
LLVM_ABI StaticDataHotness
63+
getSectionHotnessUsingDAP(std::optional<StringRef> SectionPrefix) const;
64+
6165
/// Return the string representation of the hotness enum \p Hotness.
6266
LLVM_ABI StringRef hotnessToStr(StaticDataHotness Hotness) const;
6367

68+
bool EnableDataAccessProf = false;
69+
6470
public:
65-
StaticDataProfileInfo() = default;
71+
StaticDataProfileInfo(bool EnableDataAccessProf)
72+
: EnableDataAccessProf(EnableDataAccessProf) {}
6673

6774
/// If \p Count is not nullopt, add it to the profile count of the constant \p
6875
/// C in a saturating way, and clamp the count to \p getInstrMaxCountValue if
@@ -71,14 +78,10 @@ class StaticDataProfileInfo {
7178
LLVM_ABI void addConstantProfileCount(const Constant *C,
7279
std::optional<uint64_t> Count);
7380

74-
/// Return a section prefix for the constant \p C based on its profile count.
75-
/// - If a constant doesn't have a counter, return an empty string.
76-
/// - Otherwise,
77-
/// - If it has a hot count, return "hot".
78-
/// - If it is seen by unprofiled function, return an empty string.
79-
/// - If it has a cold count, return "unlikely".
80-
/// - Otherwise (e.g. it's used by lukewarm functions), return an empty
81-
/// string.
81+
/// Given a constant \p C, returns a section prefix.
82+
/// If \p C is a global variable, the section prefix is the bigger one
83+
/// between its existing section prefix and its use profile count. Otherwise,
84+
/// the section prefix is based on its use profile count.
8285
LLVM_ABI StringRef getConstantSectionPrefix(
8386
const Constant *C, const ProfileSummaryInfo *PSI) const;
8487
};

‎llvm/lib/Analysis/StaticDataProfileInfo.cpp‎

Lines changed: 58 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
11
#include "llvm/Analysis/StaticDataProfileInfo.h"
22
#include "llvm/Analysis/ProfileSummaryInfo.h"
33
#include "llvm/IR/Constant.h"
4+
#include "llvm/IR/Constants.h"
45
#include "llvm/IR/GlobalVariable.h"
6+
#include "llvm/IR/Module.h"
57
#include "llvm/InitializePasses.h"
68
#include "llvm/ProfileData/InstrProf.h"
79

10+
#define DEBUG_TYPE "static-data-profile-info"
11+
812
using namespace llvm;
913

1014
namespace llvm {
@@ -46,6 +50,12 @@ bool IsAnnotationOK(const GlobalVariable &GV) {
4650
} // namespace memprof
4751
} // namespace llvm
4852

53+
#ifndef NDEBUG
54+
static StringRef debugPrintSectionPrefix(StringRef Prefix) {
55+
return Prefix.empty() ? "<empty>" : Prefix;
56+
}
57+
#endif
58+
4959
void StaticDataProfileInfo::addConstantProfileCount(
5060
const Constant *C, std::optional<uint64_t> Count) {
5161
if (!Count) {
@@ -79,6 +89,18 @@ StaticDataProfileInfo::getSectionHotnessUsingProfileCount(
7989
return StaticDataHotness::LukewarmOrUnknown;
8090
}
8191

92+
StaticDataProfileInfo::StaticDataHotness
93+
StaticDataProfileInfo::getSectionHotnessUsingDAP(
94+
std::optional<StringRef> MaybeSectionPrefix) const {
95+
if (!MaybeSectionPrefix)
96+
return StaticDataProfileInfo::StaticDataHotness::LukewarmOrUnknown;
97+
StringRef Prefix = *MaybeSectionPrefix;
98+
assert((Prefix == "hot" || Prefix == "unlikely") &&
99+
"Expect section_prefix to be one of hot or unlikely");
100+
return Prefix == "hot" ? StaticDataProfileInfo::StaticDataHotness::Hot
101+
: StaticDataProfileInfo::StaticDataHotness::Cold;
102+
}
103+
82104
StringRef StaticDataProfileInfo::hotnessToStr(
83105
StaticDataProfileInfo::StaticDataHotness Hotness) const {
84106
switch (Hotness) {
@@ -102,13 +124,48 @@ StaticDataProfileInfo::getConstantProfileCount(const Constant *C) const {
102124
StringRef StaticDataProfileInfo::getConstantSectionPrefix(
103125
const Constant *C, const ProfileSummaryInfo *PSI) const {
104126
std::optional<uint64_t> Count = getConstantProfileCount(C);
127+
128+
if (EnableDataAccessProf) {
129+
// Module flag `HasDataAccessProf` is 1 -> empty section prefix means
130+
// unknown hotness except for string literals.
131+
if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(C);
132+
GV && llvm::memprof::IsAnnotationOK(*GV) &&
133+
!GV->getName().starts_with(".str")) {
134+
auto HotnessFromDAP = getSectionHotnessUsingDAP(GV->getSectionPrefix());
135+
136+
if (!Count) {
137+
StringRef Prefix = hotnessToStr(HotnessFromDAP);
138+
LLVM_DEBUG(dbgs() << GV->getName() << "has section prefix "
139+
<< debugPrintSectionPrefix(Prefix)
140+
<< ", solely from data access profiles\n");
141+
return Prefix;
142+
}
143+
144+
// Both DAP and PGO counters are available. Use the hotter one.
145+
auto HotnessFromPGO = getSectionHotnessUsingProfileCount(C, PSI, *Count);
146+
StringRef Prefix = hotnessToStr(std::max(HotnessFromDAP, HotnessFromPGO));
147+
LLVM_DEBUG(dbgs() << GV->getName() << " has section prefix "
148+
<< debugPrintSectionPrefix(Prefix)
149+
<< ", the max from DAP as "
150+
<< debugPrintSectionPrefix(hotnessToStr(HotnessFromDAP))
151+
<< " and PGO counters as "
152+
<< debugPrintSectionPrefix(hotnessToStr(HotnessFromPGO))
153+
<< "\n");
154+
return Prefix;
155+
}
156+
}
105157
if (!Count)
106158
return "";
159+
107160
return hotnessToStr(getSectionHotnessUsingProfileCount(C, PSI, *Count));
108161
}
109162

110163
bool StaticDataProfileInfoWrapperPass::doInitialization(Module &M) {
111-
Info.reset(new StaticDataProfileInfo());
164+
bool EnableDataAccessProf = false;
165+
if (auto *MD = mdconst::extract_or_null<ConstantInt>(
166+
M.getModuleFlag("EnableDataAccessProf")))
167+
EnableDataAccessProf = MD->getZExtValue();
168+
Info.reset(new StaticDataProfileInfo(EnableDataAccessProf));
112169
return false;
113170
}
114171

‎llvm/test/CodeGen/X86/global-variable-partition-with-dap.ll‎

Lines changed: 93 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,102 @@
11
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
22
target triple = "x86_64-unknown-linux-gnu"
33

4-
;; A minimal test case. Subsequent PRs will expand on this test case
5-
;; (e.g., with more functions, variables and profiles) and test the hotness
6-
;; reconcillation implementation.
4+
;; Requires asserts for -debug-only.
5+
; REQUIRES: asserts
6+
7+
; RUN: rm -rf %t && split-file %s %t && cd %t
8+
9+
; RUN: llc -mtriple=x86_64-unknown-linux-gnu -relocation-model=pic \
10+
; RUN: -partition-static-data-sections=true \
11+
; RUN: -debug-only=static-data-profile-info \
12+
; RUN: -data-sections=true -unique-section-names=false \
13+
; RUN: input-with-dap-enabled.ll -o - 2>&1 | FileCheck %s --check-prefixes=LOG,IR
14+
715
; RUN: llc -mtriple=x86_64-unknown-linux-gnu -relocation-model=pic \
816
; RUN: -partition-static-data-sections=true \
17+
; RUN: -debug-only=static-data-profile-info \
918
; RUN: -data-sections=true -unique-section-names=false \
10-
; RUN: %s -o - 2>&1 | FileCheck %s --check-prefix=IR
19+
; RUN: input-without-dap.ll -o - 2>&1 | FileCheck %s --check-prefixes=NODAP
20+
21+
; LOG: hot_bss has section prefix hot, the max from DAP as hot and PGO counters as hot
22+
; LOG: data_unknown_hotness has section prefix <empty>, the max from DAP as <empty> and PGO counters as unlikely
23+
; LOG: external_relro_arrayhas section prefix unlikely, solely from data access profiles
24+
25+
; IR: .type hot_bss,@object
26+
; IR-NEXT: .section .bss.hot.,"aw"
27+
; IR: .type data_unknown_hotness,@object
28+
; IR-NEXT: .section .data,"aw"
29+
; IR: .type external_relro_array,@object
30+
; IR-NEXT: .section .data.rel.ro.unlikely.,"aw"
31+
32+
33+
; NODAP: .type hot_bss,@object
34+
; NODAP-NEXT: .section .bss.hot.,"aw"
35+
; NODAP: .type data_unknown_hotness,@object
36+
; NODAP-NEXT: .section .data.unlikely.,"aw"
37+
;; Global variable section prefix metadata is not used when
38+
;; module flag `EnableDataAccessProf` is 0, and @external_relro_array has
39+
;; external linkage, so analysis based on PGO counters doesn't apply.
40+
; NODAP: .type external_relro_array,@object # @external_relro_array
41+
; NODAP: .section .data.rel.ro,"aw"
42+
43+
;--- input-with-dap-enabled.ll
44+
; Internal vars
45+
@hot_bss = internal global i32 0, !section_prefix !17
46+
@data_unknown_hotness = internal global i32 1
47+
; External vars
48+
@external_relro_array = constant [2 x ptr] [ptr @hot_bss, ptr @data_unknown_hotness], !section_prefix !18
49+
50+
define void @cold_func() !prof !15 {
51+
%9 = load i32, ptr @data_unknown_hotness
52+
%11 = call i32 (...) @func_taking_arbitrary_param(i32 %9)
53+
ret void
54+
}
55+
56+
define void @hot_func() !prof !14 {
57+
%9 = load i32, ptr @hot_bss
58+
%11 = call i32 (...) @func_taking_arbitrary_param(i32 %9)
59+
ret void
60+
}
61+
62+
declare i32 @func_taking_arbitrary_param(...)
1163

12-
; IR: .section .bss.hot.,"aw"
64+
!llvm.module.flags = !{!0, !1}
1365

66+
!0 = !{i32 2, !"EnableDataAccessProf", i32 1}
67+
!1 = !{i32 1, !"ProfileSummary", !2}
68+
!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
69+
!3 = !{!"ProfileFormat", !"InstrProf"}
70+
!4 = !{!"TotalCount", i64 1460183}
71+
!5 = !{!"MaxCount", i64 849024}
72+
!6 = !{!"MaxInternalCount", i64 32769}
73+
!7 = !{!"MaxFunctionCount", i64 849024}
74+
!8 = !{!"NumCounts", i64 23627}
75+
!9 = !{!"NumFunctions", i64 3271}
76+
!10 = !{!"DetailedSummary", !11}
77+
!11 = !{!12, !13}
78+
!12 = !{i32 990000, i64 166, i32 73}
79+
!13 = !{i32 999999, i64 3, i32 1443}
80+
!14 = !{!"function_entry_count", i64 100000}
81+
!15 = !{!"function_entry_count", i64 1}
82+
!16 = !{!"branch_weights", i32 1, i32 99999}
83+
!17 = !{!"section_prefix", !"hot"}
84+
!18 = !{!"section_prefix", !"unlikely"}
85+
86+
;--- input-without-dap.ll
87+
; Same as `input-with-dap-enabled.ll` above except that module flag
88+
; `EnableDataAccessProf` has value 0.
89+
; Internal vars
1490
@hot_bss = internal global i32 0, !section_prefix !17
91+
@data_unknown_hotness = internal global i32 1
92+
; External vars
93+
@external_relro_array = constant [2 x ptr] [ptr @hot_bss, ptr @data_unknown_hotness], !section_prefix !18
94+
95+
define void @cold_func() !prof !15 {
96+
%9 = load i32, ptr @data_unknown_hotness
97+
%11 = call i32 (...) @func_taking_arbitrary_param(i32 %9)
98+
ret void
99+
}
15100

16101
define void @hot_func() !prof !14 {
17102
%9 = load i32, ptr @hot_bss
@@ -21,8 +106,9 @@ define void @hot_func() !prof !14 {
21106

22107
declare i32 @func_taking_arbitrary_param(...)
23108

24-
!llvm.module.flags = !{!1}
109+
!llvm.module.flags = !{!0, !1}
25110

111+
!0 = !{i32 2, !"EnableDataAccessProf", i32 0}
26112
!1 = !{i32 1, !"ProfileSummary", !2}
27113
!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
28114
!3 = !{!"ProfileFormat", !"InstrProf"}
@@ -40,3 +126,4 @@ declare i32 @func_taking_arbitrary_param(...)
40126
!15 = !{!"function_entry_count", i64 1}
41127
!16 = !{!"branch_weights", i32 1, i32 99999}
42128
!17 = !{!"section_prefix", !"hot"}
129+
!18 = !{!"section_prefix", !"unlikely"}

0 commit comments

Comments
 (0)