Skip to content

Commit 38860a8

Browse files
committed
Add names to blocks and variables
1 parent ab3b646 commit 38860a8

File tree

2 files changed

+458
-425
lines changed

2 files changed

+458
-425
lines changed

llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp

Lines changed: 66 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1184,7 +1184,7 @@ Value *LoopIdiomVectorize::expandFindFirstByte(
11841184
// Split block in the original loop preheader.
11851185
// SPH is the new preheader to the old scalar loop.
11861186
BasicBlock *SPH = SplitBlock(Preheader, Preheader->getTerminator(), DT, LI,
1187-
nullptr, "scalar_ph");
1187+
nullptr, "scalar_preheader");
11881188

11891189
// Create the blocks that we're going to use.
11901190
//
@@ -1206,12 +1206,17 @@ Value *LoopIdiomVectorize::expandFindFirstByte(
12061206
// (1), otherwise exit.
12071207
// Blocks (0,3) are not part of any loop. Blocks (1,5) and (2,4) belong to
12081208
// the outer and inner loops, respectively.
1209-
BasicBlock *BB0 = BasicBlock::Create(Ctx, "", SPH->getParent(), SPH);
1210-
BasicBlock *BB1 = BasicBlock::Create(Ctx, "", SPH->getParent(), SPH);
1211-
BasicBlock *BB2 = BasicBlock::Create(Ctx, "", SPH->getParent(), SPH);
1212-
BasicBlock *BB3 = BasicBlock::Create(Ctx, "", SPH->getParent(), SPH);
1213-
BasicBlock *BB4 = BasicBlock::Create(Ctx, "", SPH->getParent(), SPH);
1214-
BasicBlock *BB5 = BasicBlock::Create(Ctx, "", SPH->getParent(), SPH);
1209+
BasicBlock *BB0 = BasicBlock::Create(Ctx, "mem_check", SPH->getParent(), SPH);
1210+
BasicBlock *BB1 =
1211+
BasicBlock::Create(Ctx, "find_first_vec_header", SPH->getParent(), SPH);
1212+
BasicBlock *BB2 =
1213+
BasicBlock::Create(Ctx, "match_check_vec", SPH->getParent(), SPH);
1214+
BasicBlock *BB3 =
1215+
BasicBlock::Create(Ctx, "calculate_match", SPH->getParent(), SPH);
1216+
BasicBlock *BB4 =
1217+
BasicBlock::Create(Ctx, "needle_check_vec", SPH->getParent(), SPH);
1218+
BasicBlock *BB5 =
1219+
BasicBlock::Create(Ctx, "search_check_vec", SPH->getParent(), SPH);
12151220

12161221
// Update LoopInfo with the new loops.
12171222
auto OuterLoop = LI->AllocateLoop();
@@ -1243,24 +1248,35 @@ Value *LoopIdiomVectorize::expandFindFirstByte(
12431248
// old scalar loops. Also create a predicate of VF elements to be used in the
12441249
// vector loops.
12451250
Builder.SetInsertPoint(BB0);
1246-
Value *ISearchStart = Builder.CreatePtrToInt(SearchStart, I64Ty);
1247-
Value *ISearchEnd = Builder.CreatePtrToInt(SearchEnd, I64Ty);
1248-
Value *INeedleStart = Builder.CreatePtrToInt(NeedleStart, I64Ty);
1249-
Value *INeedleEnd = Builder.CreatePtrToInt(NeedleEnd, I64Ty);
1251+
Value *ISearchStart =
1252+
Builder.CreatePtrToInt(SearchStart, I64Ty, "search_start_int");
1253+
Value *ISearchEnd =
1254+
Builder.CreatePtrToInt(SearchEnd, I64Ty, "search_end_int");
1255+
Value *INeedleStart =
1256+
Builder.CreatePtrToInt(NeedleStart, I64Ty, "needle_start_int");
1257+
Value *INeedleEnd =
1258+
Builder.CreatePtrToInt(NeedleEnd, I64Ty, "needle_end_int");
12501259
Value *PredVF =
12511260
Builder.CreateIntrinsic(Intrinsic::get_active_lane_mask, {PredVTy, I64Ty},
12521261
{ConstantInt::get(I64Ty, 0), ConstVF});
12531262

12541263
const uint64_t MinPageSize = TTI->getMinPageSize().value();
12551264
const uint64_t AddrShiftAmt = llvm::Log2_64(MinPageSize);
1256-
Value *SearchStartPage = Builder.CreateLShr(ISearchStart, AddrShiftAmt);
1257-
Value *SearchEndPage = Builder.CreateLShr(ISearchEnd, AddrShiftAmt);
1258-
Value *NeedleStartPage = Builder.CreateLShr(INeedleStart, AddrShiftAmt);
1259-
Value *NeedleEndPage = Builder.CreateLShr(INeedleEnd, AddrShiftAmt);
1260-
Value *SearchPageCmp = Builder.CreateICmpNE(SearchStartPage, SearchEndPage);
1261-
Value *NeedlePageCmp = Builder.CreateICmpNE(NeedleStartPage, NeedleEndPage);
1262-
1263-
Value *CombinedPageCmp = Builder.CreateOr(SearchPageCmp, NeedlePageCmp);
1265+
Value *SearchStartPage =
1266+
Builder.CreateLShr(ISearchStart, AddrShiftAmt, "search_start_page");
1267+
Value *SearchEndPage =
1268+
Builder.CreateLShr(ISearchEnd, AddrShiftAmt, "search_end_page");
1269+
Value *NeedleStartPage =
1270+
Builder.CreateLShr(INeedleStart, AddrShiftAmt, "needle_start_page");
1271+
Value *NeedleEndPage =
1272+
Builder.CreateLShr(INeedleEnd, AddrShiftAmt, "needle_end_page");
1273+
Value *SearchPageCmp =
1274+
Builder.CreateICmpNE(SearchStartPage, SearchEndPage, "search_page_cmp");
1275+
Value *NeedlePageCmp =
1276+
Builder.CreateICmpNE(NeedleStartPage, NeedleEndPage, "needle_page_cmp");
1277+
1278+
Value *CombinedPageCmp =
1279+
Builder.CreateOr(SearchPageCmp, NeedlePageCmp, "combined_page_cmp");
12641280
BranchInst *CombinedPageBr = Builder.CreateCondBr(CombinedPageCmp, SPH, BB1);
12651281
CombinedPageBr->setMetadata(LLVMContext::MD_prof,
12661282
MDBuilder(Ctx).createBranchWeights(10, 90));
@@ -1272,10 +1288,11 @@ Value *LoopIdiomVectorize::expandFindFirstByte(
12721288
PHINode *Search = Builder.CreatePHI(PtrTy, 2, "psearch");
12731289
Value *PredSearch = Builder.CreateIntrinsic(
12741290
Intrinsic::get_active_lane_mask, {PredVTy, I64Ty},
1275-
{Builder.CreatePtrToInt(Search, I64Ty), ISearchEnd});
1276-
PredSearch = Builder.CreateAnd(PredVF, PredSearch);
1277-
Value *LoadSearch =
1278-
Builder.CreateMaskedLoad(CharVTy, Search, Align(1), PredSearch, Passthru);
1291+
{Builder.CreatePtrToInt(Search, I64Ty), ISearchEnd}, nullptr,
1292+
"search_pred");
1293+
PredSearch = Builder.CreateAnd(PredVF, PredSearch, "search_masked");
1294+
Value *LoadSearch = Builder.CreateMaskedLoad(
1295+
CharVTy, Search, Align(1), PredSearch, Passthru, "search_load_vec");
12791296
Builder.CreateBr(BB2);
12801297
DTU.applyUpdates({{DominatorTree::Insert, BB1, BB2}});
12811298

@@ -1286,49 +1303,58 @@ Value *LoopIdiomVectorize::expandFindFirstByte(
12861303
// (2.a) Load the needle array.
12871304
Value *PredNeedle = Builder.CreateIntrinsic(
12881305
Intrinsic::get_active_lane_mask, {PredVTy, I64Ty},
1289-
{Builder.CreatePtrToInt(Needle, I64Ty), INeedleEnd});
1290-
PredNeedle = Builder.CreateAnd(PredVF, PredNeedle);
1291-
Value *LoadNeedle =
1292-
Builder.CreateMaskedLoad(CharVTy, Needle, Align(1), PredNeedle, Passthru);
1306+
{Builder.CreatePtrToInt(Needle, I64Ty), INeedleEnd}, nullptr,
1307+
"needle_pred");
1308+
PredNeedle = Builder.CreateAnd(PredVF, PredNeedle, "needle_masked");
1309+
Value *LoadNeedle = Builder.CreateMaskedLoad(
1310+
CharVTy, Needle, Align(1), PredNeedle, Passthru, "needle_load_vec");
12931311

12941312
// (2.b) Splat the first element to the inactive lanes.
1295-
Value *Needle0 = Builder.CreateExtractElement(LoadNeedle, uint64_t(0));
1296-
Value *Needle0Splat =
1297-
Builder.CreateVectorSplat(ElementCount::getScalable(VF), Needle0);
1298-
LoadNeedle = Builder.CreateSelect(PredNeedle, LoadNeedle, Needle0Splat);
1299-
LoadNeedle = Builder.CreateExtractVector(
1300-
FixedVectorType::get(CharTy, VF), LoadNeedle, ConstantInt::get(I64Ty, 0));
1313+
Value *Needle0 =
1314+
Builder.CreateExtractElement(LoadNeedle, uint64_t(0), "needle0");
1315+
Value *Needle0Splat = Builder.CreateVectorSplat(ElementCount::getScalable(VF),
1316+
Needle0, "needle0");
1317+
LoadNeedle = Builder.CreateSelect(PredNeedle, LoadNeedle, Needle0Splat,
1318+
"needle_splat");
1319+
LoadNeedle =
1320+
Builder.CreateExtractVector(FixedVectorType::get(CharTy, VF), LoadNeedle,
1321+
ConstantInt::get(I64Ty, 0), "needle_vec");
13011322

13021323
// (2.c) Test if there's a match.
13031324
Value *MatchPred = Builder.CreateIntrinsic(
13041325
Intrinsic::experimental_vector_match, {CharVTy, LoadNeedle->getType()},
1305-
{LoadSearch, LoadNeedle, PredSearch});
1326+
{LoadSearch, LoadNeedle, PredSearch}, nullptr, "match_pred");
13061327
Value *IfAnyMatch = Builder.CreateOrReduce(MatchPred);
13071328
Builder.CreateCondBr(IfAnyMatch, BB3, BB4);
13081329
DTU.applyUpdates(
13091330
{{DominatorTree::Insert, BB2, BB3}, {DominatorTree::Insert, BB2, BB4}});
13101331

13111332
// (3) We found a match. Compute the index of its location and exit.
13121333
Builder.SetInsertPoint(BB3);
1313-
PHINode *MatchLCSSA = Builder.CreatePHI(PtrTy, 1);
1314-
PHINode *MatchPredLCSSA = Builder.CreatePHI(MatchPred->getType(), 1);
1334+
PHINode *MatchLCSSA = Builder.CreatePHI(PtrTy, 1, "match_start");
1335+
PHINode *MatchPredLCSSA =
1336+
Builder.CreatePHI(MatchPred->getType(), 1, "match_vec");
13151337
Value *MatchCnt = Builder.CreateIntrinsic(
13161338
Intrinsic::experimental_cttz_elts, {I64Ty, MatchPred->getType()},
1317-
{MatchPredLCSSA, /*ZeroIsPoison=*/Builder.getInt1(true)});
1318-
Value *MatchVal = Builder.CreateGEP(CharTy, MatchLCSSA, MatchCnt);
1339+
{MatchPredLCSSA, /*ZeroIsPoison=*/Builder.getInt1(true)}, nullptr,
1340+
"match_idx");
1341+
Value *MatchVal =
1342+
Builder.CreateGEP(CharTy, MatchLCSSA, MatchCnt, "match_res");
13191343
Builder.CreateBr(ExitSucc);
13201344
DTU.applyUpdates({{DominatorTree::Insert, BB3, ExitSucc}});
13211345

13221346
// (4) Check if we've reached the end of the needle array.
13231347
Builder.SetInsertPoint(BB4);
1324-
Value *NextNeedle = Builder.CreateGEP(CharTy, Needle, ConstVF);
1348+
Value *NextNeedle =
1349+
Builder.CreateGEP(CharTy, Needle, ConstVF, "needle_next_vec");
13251350
Builder.CreateCondBr(Builder.CreateICmpULT(NextNeedle, NeedleEnd), BB2, BB5);
13261351
DTU.applyUpdates(
13271352
{{DominatorTree::Insert, BB4, BB2}, {DominatorTree::Insert, BB4, BB5}});
13281353

13291354
// (5) Check if we've reached the end of the search array.
13301355
Builder.SetInsertPoint(BB5);
1331-
Value *NextSearch = Builder.CreateGEP(CharTy, Search, ConstVF);
1356+
Value *NextSearch =
1357+
Builder.CreateGEP(CharTy, Search, ConstVF, "search_next_vec");
13321358
Builder.CreateCondBr(Builder.CreateICmpULT(NextSearch, SearchEnd), BB1,
13331359
ExitFail);
13341360
DTU.applyUpdates({{DominatorTree::Insert, BB5, BB1},

0 commit comments

Comments
 (0)