24
24
25
25
using namespace llvm ;
26
26
27
- namespace {
28
-
29
27
// MD_prof nodes have the following layout
30
28
//
31
29
// In general:
@@ -41,14 +39,15 @@ namespace {
41
39
// correctly, and can change the behavior in the future if the layout changes
42
40
43
41
// the minimum number of operands for MD_prof nodes with branch weights
44
- constexpr unsigned MinBWOps = 3 ;
42
+ static constexpr unsigned MinBWOps = 3 ;
45
43
46
44
// the minimum number of operands for MD_prof nodes with value profiles
47
- constexpr unsigned MinVPOps = 5 ;
45
+ static constexpr unsigned MinVPOps = 5 ;
48
46
49
47
// We may want to add support for other MD_prof types, so provide an abstraction
50
48
// for checking the metadata type.
51
- bool isTargetMD (const MDNode *ProfData, const char *Name, unsigned MinOps) {
49
+ static bool isTargetMD (const MDNode *ProfData, const char *Name,
50
+ unsigned MinOps) {
52
51
// TODO: This routine may be simplified if MD_prof used an enum instead of a
53
52
// string to differentiate the types of MD_prof nodes.
54
53
if (!ProfData || !Name || MinOps < 2 )
@@ -101,14 +100,11 @@ static SmallVector<uint32_t> fitWeights(ArrayRef<uint64_t> Weights) {
101
100
return Ret;
102
101
}
103
102
104
- } // namespace
105
-
106
- namespace llvm {
107
- cl::opt<bool > ElideAllZeroBranchWeights (" elide-all-zero-branch-weights" ,
103
+ static cl::opt<bool > ElideAllZeroBranchWeights (" elide-all-zero-branch-weights" ,
108
104
#if defined(LLVM_ENABLE_PROFCHECK)
109
- cl::init (false )
105
+ cl::init (false )
110
106
#else
111
- cl::init (true )
107
+ cl::init (true )
112
108
#endif
113
109
);
114
110
const char *MDProfLabels::BranchWeights = " branch_weights" ;
@@ -118,21 +114,21 @@ const char *MDProfLabels::FunctionEntryCount = "function_entry_count";
118
114
const char *MDProfLabels::SyntheticFunctionEntryCount =
119
115
" synthetic_function_entry_count" ;
120
116
const char *MDProfLabels::UnknownBranchWeightsMarker = " unknown" ;
121
- const char *LLVMLoopEstimatedTripCount = " llvm.loop.estimated_trip_count" ;
117
+ const char *llvm:: LLVMLoopEstimatedTripCount = " llvm.loop.estimated_trip_count" ;
122
118
123
- bool hasProfMD (const Instruction &I) {
119
+ bool llvm:: hasProfMD (const Instruction &I) {
124
120
return I.hasMetadata (LLVMContext::MD_prof);
125
121
}
126
122
127
- bool isBranchWeightMD (const MDNode *ProfileData) {
123
+ bool llvm:: isBranchWeightMD (const MDNode *ProfileData) {
128
124
return isTargetMD (ProfileData, MDProfLabels::BranchWeights, MinBWOps);
129
125
}
130
126
131
- bool isValueProfileMD (const MDNode *ProfileData) {
127
+ bool llvm:: isValueProfileMD (const MDNode *ProfileData) {
132
128
return isTargetMD (ProfileData, MDProfLabels::ValueProfile, MinVPOps);
133
129
}
134
130
135
- bool hasBranchWeightMD (const Instruction &I) {
131
+ bool llvm:: hasBranchWeightMD (const Instruction &I) {
136
132
auto *ProfileData = I.getMetadata (LLVMContext::MD_prof);
137
133
return isBranchWeightMD (ProfileData);
138
134
}
@@ -147,16 +143,16 @@ static bool hasCountTypeMD(const Instruction &I) {
147
143
return isa<CallBase>(I) && !isBranchWeightMD (ProfileData);
148
144
}
149
145
150
- bool hasValidBranchWeightMD (const Instruction &I) {
146
+ bool llvm:: hasValidBranchWeightMD (const Instruction &I) {
151
147
return getValidBranchWeightMDNode (I);
152
148
}
153
149
154
- bool hasBranchWeightOrigin (const Instruction &I) {
150
+ bool llvm:: hasBranchWeightOrigin (const Instruction &I) {
155
151
auto *ProfileData = I.getMetadata (LLVMContext::MD_prof);
156
152
return hasBranchWeightOrigin (ProfileData);
157
153
}
158
154
159
- bool hasBranchWeightOrigin (const MDNode *ProfileData) {
155
+ bool llvm:: hasBranchWeightOrigin (const MDNode *ProfileData) {
160
156
if (!isBranchWeightMD (ProfileData))
161
157
return false ;
162
158
auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand (1 ));
@@ -168,54 +164,54 @@ bool hasBranchWeightOrigin(const MDNode *ProfileData) {
168
164
return ProfDataName != nullptr ;
169
165
}
170
166
171
- unsigned getBranchWeightOffset (const MDNode *ProfileData) {
167
+ unsigned llvm:: getBranchWeightOffset (const MDNode *ProfileData) {
172
168
return hasBranchWeightOrigin (ProfileData) ? 2 : 1 ;
173
169
}
174
170
175
- unsigned getNumBranchWeights (const MDNode &ProfileData) {
171
+ unsigned llvm:: getNumBranchWeights (const MDNode &ProfileData) {
176
172
return ProfileData.getNumOperands () - getBranchWeightOffset (&ProfileData);
177
173
}
178
174
179
- MDNode *getBranchWeightMDNode (const Instruction &I) {
175
+ MDNode *llvm:: getBranchWeightMDNode (const Instruction &I) {
180
176
auto *ProfileData = I.getMetadata (LLVMContext::MD_prof);
181
177
if (!isBranchWeightMD (ProfileData))
182
178
return nullptr ;
183
179
return ProfileData;
184
180
}
185
181
186
- MDNode *getValidBranchWeightMDNode (const Instruction &I) {
182
+ MDNode *llvm:: getValidBranchWeightMDNode (const Instruction &I) {
187
183
auto *ProfileData = getBranchWeightMDNode (I);
188
184
if (ProfileData && getNumBranchWeights (*ProfileData) == I.getNumSuccessors ())
189
185
return ProfileData;
190
186
return nullptr ;
191
187
}
192
188
193
- void extractFromBranchWeightMD32 (const MDNode *ProfileData,
194
- SmallVectorImpl<uint32_t > &Weights) {
189
+ void llvm:: extractFromBranchWeightMD32 (const MDNode *ProfileData,
190
+ SmallVectorImpl<uint32_t > &Weights) {
195
191
extractFromBranchWeightMD (ProfileData, Weights);
196
192
}
197
193
198
- void extractFromBranchWeightMD64 (const MDNode *ProfileData,
199
- SmallVectorImpl<uint64_t > &Weights) {
194
+ void llvm:: extractFromBranchWeightMD64 (const MDNode *ProfileData,
195
+ SmallVectorImpl<uint64_t > &Weights) {
200
196
extractFromBranchWeightMD (ProfileData, Weights);
201
197
}
202
198
203
- bool extractBranchWeights (const MDNode *ProfileData,
204
- SmallVectorImpl<uint32_t > &Weights) {
199
+ bool llvm:: extractBranchWeights (const MDNode *ProfileData,
200
+ SmallVectorImpl<uint32_t > &Weights) {
205
201
if (!isBranchWeightMD (ProfileData))
206
202
return false ;
207
203
extractFromBranchWeightMD (ProfileData, Weights);
208
204
return true ;
209
205
}
210
206
211
- bool extractBranchWeights (const Instruction &I,
212
- SmallVectorImpl<uint32_t > &Weights) {
207
+ bool llvm:: extractBranchWeights (const Instruction &I,
208
+ SmallVectorImpl<uint32_t > &Weights) {
213
209
auto *ProfileData = I.getMetadata (LLVMContext::MD_prof);
214
210
return extractBranchWeights (ProfileData, Weights);
215
211
}
216
212
217
- bool extractBranchWeights (const Instruction &I, uint64_t &TrueVal,
218
- uint64_t &FalseVal) {
213
+ bool llvm:: extractBranchWeights (const Instruction &I, uint64_t &TrueVal,
214
+ uint64_t &FalseVal) {
219
215
assert ((I.getOpcode () == Instruction::Br ||
220
216
I.getOpcode () == Instruction::Select) &&
221
217
" Looking for branch weights on something besides branch, select, or "
@@ -234,7 +230,8 @@ bool extractBranchWeights(const Instruction &I, uint64_t &TrueVal,
234
230
return true ;
235
231
}
236
232
237
- bool extractProfTotalWeight (const MDNode *ProfileData, uint64_t &TotalVal) {
233
+ bool llvm::extractProfTotalWeight (const MDNode *ProfileData,
234
+ uint64_t &TotalVal) {
238
235
TotalVal = 0 ;
239
236
if (!ProfileData)
240
237
return false ;
@@ -262,11 +259,12 @@ bool extractProfTotalWeight(const MDNode *ProfileData, uint64_t &TotalVal) {
262
259
return false ;
263
260
}
264
261
265
- bool extractProfTotalWeight (const Instruction &I, uint64_t &TotalVal) {
262
+ bool llvm:: extractProfTotalWeight (const Instruction &I, uint64_t &TotalVal) {
266
263
return extractProfTotalWeight (I.getMetadata (LLVMContext::MD_prof), TotalVal);
267
264
}
268
265
269
- void setExplicitlyUnknownBranchWeights (Instruction &I, StringRef PassName) {
266
+ void llvm::setExplicitlyUnknownBranchWeights (Instruction &I,
267
+ StringRef PassName) {
270
268
MDBuilder MDB (I.getContext ());
271
269
I.setMetadata (
272
270
LLVMContext::MD_prof,
@@ -275,14 +273,16 @@ void setExplicitlyUnknownBranchWeights(Instruction &I, StringRef PassName) {
275
273
MDB.createString (PassName)}));
276
274
}
277
275
278
- void setExplicitlyUnknownBranchWeightsIfProfiled (Instruction &I, Function &F,
279
- StringRef PassName) {
276
+ void llvm::setExplicitlyUnknownBranchWeightsIfProfiled (Instruction &I,
277
+ Function &F,
278
+ StringRef PassName) {
280
279
if (std::optional<Function::ProfileCount> EC = F.getEntryCount ();
281
280
EC && EC->getCount () > 0 )
282
281
setExplicitlyUnknownBranchWeights (I, PassName);
283
282
}
284
283
285
- void setExplicitlyUnknownFunctionEntryCount (Function &F, StringRef PassName) {
284
+ void llvm::setExplicitlyUnknownFunctionEntryCount (Function &F,
285
+ StringRef PassName) {
286
286
MDBuilder MDB (F.getContext ());
287
287
F.setMetadata (
288
288
LLVMContext::MD_prof,
@@ -291,21 +291,21 @@ void setExplicitlyUnknownFunctionEntryCount(Function &F, StringRef PassName) {
291
291
MDB.createString (PassName)}));
292
292
}
293
293
294
- bool isExplicitlyUnknownProfileMetadata (const MDNode &MD) {
294
+ bool llvm:: isExplicitlyUnknownProfileMetadata (const MDNode &MD) {
295
295
if (MD.getNumOperands () != 2 )
296
296
return false ;
297
297
return MD.getOperand (0 ).equalsStr (MDProfLabels::UnknownBranchWeightsMarker);
298
298
}
299
299
300
- bool hasExplicitlyUnknownBranchWeights (const Instruction &I) {
300
+ bool llvm:: hasExplicitlyUnknownBranchWeights (const Instruction &I) {
301
301
auto *MD = I.getMetadata (LLVMContext::MD_prof);
302
302
if (!MD)
303
303
return false ;
304
304
return isExplicitlyUnknownProfileMetadata (*MD);
305
305
}
306
306
307
- void setBranchWeights (Instruction &I, ArrayRef<uint32_t > Weights,
308
- bool IsExpected, bool ElideAllZero) {
307
+ void llvm:: setBranchWeights (Instruction &I, ArrayRef<uint32_t > Weights,
308
+ bool IsExpected, bool ElideAllZero) {
309
309
if ((ElideAllZeroBranchWeights && ElideAllZero) &&
310
310
llvm::all_of (Weights, [](uint32_t V) { return V == 0 ; })) {
311
311
I.setMetadata (LLVMContext::MD_prof, nullptr );
@@ -317,13 +317,14 @@ void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights,
317
317
I.setMetadata (LLVMContext::MD_prof, BranchWeights);
318
318
}
319
319
320
- void setFittedBranchWeights (Instruction &I, ArrayRef<uint64_t > Weights,
321
- bool IsExpected, bool ElideAllZero) {
320
+ void llvm:: setFittedBranchWeights (Instruction &I, ArrayRef<uint64_t > Weights,
321
+ bool IsExpected, bool ElideAllZero) {
322
322
setBranchWeights (I, fitWeights (Weights), IsExpected, ElideAllZero);
323
323
}
324
324
325
- SmallVector<uint32_t > downscaleWeights (ArrayRef<uint64_t > Weights,
326
- std::optional<uint64_t > KnownMaxCount) {
325
+ SmallVector<uint32_t >
326
+ llvm::downscaleWeights (ArrayRef<uint64_t > Weights,
327
+ std::optional<uint64_t > KnownMaxCount) {
327
328
uint64_t MaxCount = KnownMaxCount.has_value () ? KnownMaxCount.value ()
328
329
: *llvm::max_element (Weights);
329
330
assert (MaxCount > 0 && " Bad max count" );
@@ -334,7 +335,7 @@ SmallVector<uint32_t> downscaleWeights(ArrayRef<uint64_t> Weights,
334
335
return DownscaledWeights;
335
336
}
336
337
337
- void scaleProfData (Instruction &I, uint64_t S, uint64_t T) {
338
+ void llvm:: scaleProfData (Instruction &I, uint64_t S, uint64_t T) {
338
339
assert (T != 0 && " Caller should guarantee" );
339
340
auto *ProfileData = I.getMetadata (LLVMContext::MD_prof);
340
341
if (ProfileData == nullptr )
@@ -387,5 +388,3 @@ void scaleProfData(Instruction &I, uint64_t S, uint64_t T) {
387
388
}
388
389
I.setMetadata (LLVMContext::MD_prof, MDNode::get (C, Vals));
389
390
}
390
-
391
- } // namespace llvm
0 commit comments