@@ -1184,7 +1184,7 @@ Value *LoopIdiomVectorize::expandFindFirstByte(
11841184 // Split block in the original loop preheader.
11851185 // SPH is the new preheader to the old scalar loop.
11861186 BasicBlock *SPH = SplitBlock (Preheader, Preheader->getTerminator (), DT, LI,
1187- nullptr , " scalar_ph " );
1187+ nullptr , " scalar_preheader " );
11881188
11891189 // Create the blocks that we're going to use.
11901190 //
@@ -1206,12 +1206,17 @@ Value *LoopIdiomVectorize::expandFindFirstByte(
12061206 // (1), otherwise exit.
12071207 // Blocks (0,3) are not part of any loop. Blocks (1,5) and (2,4) belong to
12081208 // the outer and inner loops, respectively.
1209- BasicBlock *BB0 = BasicBlock::Create (Ctx, " " , SPH->getParent (), SPH);
1210- BasicBlock *BB1 = BasicBlock::Create (Ctx, " " , SPH->getParent (), SPH);
1211- BasicBlock *BB2 = BasicBlock::Create (Ctx, " " , SPH->getParent (), SPH);
1212- BasicBlock *BB3 = BasicBlock::Create (Ctx, " " , SPH->getParent (), SPH);
1213- BasicBlock *BB4 = BasicBlock::Create (Ctx, " " , SPH->getParent (), SPH);
1214- BasicBlock *BB5 = BasicBlock::Create (Ctx, " " , SPH->getParent (), SPH);
1209+ BasicBlock *BB0 = BasicBlock::Create (Ctx, " mem_check" , SPH->getParent (), SPH);
1210+ BasicBlock *BB1 =
1211+ BasicBlock::Create (Ctx, " find_first_vec_header" , SPH->getParent (), SPH);
1212+ BasicBlock *BB2 =
1213+ BasicBlock::Create (Ctx, " match_check_vec" , SPH->getParent (), SPH);
1214+ BasicBlock *BB3 =
1215+ BasicBlock::Create (Ctx, " calculate_match" , SPH->getParent (), SPH);
1216+ BasicBlock *BB4 =
1217+ BasicBlock::Create (Ctx, " needle_check_vec" , SPH->getParent (), SPH);
1218+ BasicBlock *BB5 =
1219+ BasicBlock::Create (Ctx, " search_check_vec" , SPH->getParent (), SPH);
12151220
12161221 // Update LoopInfo with the new loops.
12171222 auto OuterLoop = LI->AllocateLoop ();
@@ -1243,24 +1248,35 @@ Value *LoopIdiomVectorize::expandFindFirstByte(
12431248 // old scalar loops. Also create a predicate of VF elements to be used in the
12441249 // vector loops.
12451250 Builder.SetInsertPoint (BB0);
1246- Value *ISearchStart = Builder.CreatePtrToInt (SearchStart, I64Ty);
1247- Value *ISearchEnd = Builder.CreatePtrToInt (SearchEnd, I64Ty);
1248- Value *INeedleStart = Builder.CreatePtrToInt (NeedleStart, I64Ty);
1249- Value *INeedleEnd = Builder.CreatePtrToInt (NeedleEnd, I64Ty);
1251+ Value *ISearchStart =
1252+ Builder.CreatePtrToInt (SearchStart, I64Ty, " search_start_int" );
1253+ Value *ISearchEnd =
1254+ Builder.CreatePtrToInt (SearchEnd, I64Ty, " search_end_int" );
1255+ Value *INeedleStart =
1256+ Builder.CreatePtrToInt (NeedleStart, I64Ty, " needle_start_int" );
1257+ Value *INeedleEnd =
1258+ Builder.CreatePtrToInt (NeedleEnd, I64Ty, " needle_end_int" );
12501259 Value *PredVF =
12511260 Builder.CreateIntrinsic (Intrinsic::get_active_lane_mask, {PredVTy, I64Ty},
12521261 {ConstantInt::get (I64Ty, 0 ), ConstVF});
12531262
12541263 const uint64_t MinPageSize = TTI->getMinPageSize ().value ();
12551264 const uint64_t AddrShiftAmt = llvm::Log2_64 (MinPageSize);
1256- Value *SearchStartPage = Builder.CreateLShr (ISearchStart, AddrShiftAmt);
1257- Value *SearchEndPage = Builder.CreateLShr (ISearchEnd, AddrShiftAmt);
1258- Value *NeedleStartPage = Builder.CreateLShr (INeedleStart, AddrShiftAmt);
1259- Value *NeedleEndPage = Builder.CreateLShr (INeedleEnd, AddrShiftAmt);
1260- Value *SearchPageCmp = Builder.CreateICmpNE (SearchStartPage, SearchEndPage);
1261- Value *NeedlePageCmp = Builder.CreateICmpNE (NeedleStartPage, NeedleEndPage);
1262-
1263- Value *CombinedPageCmp = Builder.CreateOr (SearchPageCmp, NeedlePageCmp);
1265+ Value *SearchStartPage =
1266+ Builder.CreateLShr (ISearchStart, AddrShiftAmt, " search_start_page" );
1267+ Value *SearchEndPage =
1268+ Builder.CreateLShr (ISearchEnd, AddrShiftAmt, " search_end_page" );
1269+ Value *NeedleStartPage =
1270+ Builder.CreateLShr (INeedleStart, AddrShiftAmt, " needle_start_page" );
1271+ Value *NeedleEndPage =
1272+ Builder.CreateLShr (INeedleEnd, AddrShiftAmt, " needle_end_page" );
1273+ Value *SearchPageCmp =
1274+ Builder.CreateICmpNE (SearchStartPage, SearchEndPage, " search_page_cmp" );
1275+ Value *NeedlePageCmp =
1276+ Builder.CreateICmpNE (NeedleStartPage, NeedleEndPage, " needle_page_cmp" );
1277+
1278+ Value *CombinedPageCmp =
1279+ Builder.CreateOr (SearchPageCmp, NeedlePageCmp, " combined_page_cmp" );
12641280 BranchInst *CombinedPageBr = Builder.CreateCondBr (CombinedPageCmp, SPH, BB1);
12651281 CombinedPageBr->setMetadata (LLVMContext::MD_prof,
12661282 MDBuilder (Ctx).createBranchWeights (10 , 90 ));
@@ -1272,10 +1288,11 @@ Value *LoopIdiomVectorize::expandFindFirstByte(
12721288 PHINode *Search = Builder.CreatePHI (PtrTy, 2 , " psearch" );
12731289 Value *PredSearch = Builder.CreateIntrinsic (
12741290 Intrinsic::get_active_lane_mask, {PredVTy, I64Ty},
1275- {Builder.CreatePtrToInt (Search, I64Ty), ISearchEnd});
1276- PredSearch = Builder.CreateAnd (PredVF, PredSearch);
1277- Value *LoadSearch =
1278- Builder.CreateMaskedLoad (CharVTy, Search, Align (1 ), PredSearch, Passthru);
1291+ {Builder.CreatePtrToInt (Search, I64Ty), ISearchEnd}, nullptr ,
1292+ " search_pred" );
1293+ PredSearch = Builder.CreateAnd (PredVF, PredSearch, " search_masked" );
1294+ Value *LoadSearch = Builder.CreateMaskedLoad (
1295+ CharVTy, Search, Align (1 ), PredSearch, Passthru, " search_load_vec" );
12791296 Builder.CreateBr (BB2);
12801297 DTU.applyUpdates ({{DominatorTree::Insert, BB1, BB2}});
12811298
@@ -1286,49 +1303,58 @@ Value *LoopIdiomVectorize::expandFindFirstByte(
12861303 // (2.a) Load the needle array.
12871304 Value *PredNeedle = Builder.CreateIntrinsic (
12881305 Intrinsic::get_active_lane_mask, {PredVTy, I64Ty},
1289- {Builder.CreatePtrToInt (Needle, I64Ty), INeedleEnd});
1290- PredNeedle = Builder.CreateAnd (PredVF, PredNeedle);
1291- Value *LoadNeedle =
1292- Builder.CreateMaskedLoad (CharVTy, Needle, Align (1 ), PredNeedle, Passthru);
1306+ {Builder.CreatePtrToInt (Needle, I64Ty), INeedleEnd}, nullptr ,
1307+ " needle_pred" );
1308+ PredNeedle = Builder.CreateAnd (PredVF, PredNeedle, " needle_masked" );
1309+ Value *LoadNeedle = Builder.CreateMaskedLoad (
1310+ CharVTy, Needle, Align (1 ), PredNeedle, Passthru, " needle_load_vec" );
12931311
12941312 // (2.b) Splat the first element to the inactive lanes.
1295- Value *Needle0 = Builder.CreateExtractElement (LoadNeedle, uint64_t (0 ));
1296- Value *Needle0Splat =
1297- Builder.CreateVectorSplat (ElementCount::getScalable (VF), Needle0);
1298- LoadNeedle = Builder.CreateSelect (PredNeedle, LoadNeedle, Needle0Splat);
1299- LoadNeedle = Builder.CreateExtractVector (
1300- FixedVectorType::get (CharTy, VF), LoadNeedle, ConstantInt::get (I64Ty, 0 ));
1313+ Value *Needle0 =
1314+ Builder.CreateExtractElement (LoadNeedle, uint64_t (0 ), " needle0" );
1315+ Value *Needle0Splat = Builder.CreateVectorSplat (ElementCount::getScalable (VF),
1316+ Needle0, " needle0" );
1317+ LoadNeedle = Builder.CreateSelect (PredNeedle, LoadNeedle, Needle0Splat,
1318+ " needle_splat" );
1319+ LoadNeedle =
1320+ Builder.CreateExtractVector (FixedVectorType::get (CharTy, VF), LoadNeedle,
1321+ ConstantInt::get (I64Ty, 0 ), " needle_vec" );
13011322
13021323 // (2.c) Test if there's a match.
13031324 Value *MatchPred = Builder.CreateIntrinsic (
13041325 Intrinsic::experimental_vector_match, {CharVTy, LoadNeedle->getType ()},
1305- {LoadSearch, LoadNeedle, PredSearch});
1326+ {LoadSearch, LoadNeedle, PredSearch}, nullptr , " match_pred " );
13061327 Value *IfAnyMatch = Builder.CreateOrReduce (MatchPred);
13071328 Builder.CreateCondBr (IfAnyMatch, BB3, BB4);
13081329 DTU.applyUpdates (
13091330 {{DominatorTree::Insert, BB2, BB3}, {DominatorTree::Insert, BB2, BB4}});
13101331
13111332 // (3) We found a match. Compute the index of its location and exit.
13121333 Builder.SetInsertPoint (BB3);
1313- PHINode *MatchLCSSA = Builder.CreatePHI (PtrTy, 1 );
1314- PHINode *MatchPredLCSSA = Builder.CreatePHI (MatchPred->getType (), 1 );
1334+ PHINode *MatchLCSSA = Builder.CreatePHI (PtrTy, 1 , " match_start" );
1335+ PHINode *MatchPredLCSSA =
1336+ Builder.CreatePHI (MatchPred->getType (), 1 , " match_vec" );
13151337 Value *MatchCnt = Builder.CreateIntrinsic (
13161338 Intrinsic::experimental_cttz_elts, {I64Ty, MatchPred->getType ()},
1317- {MatchPredLCSSA, /* ZeroIsPoison=*/ Builder.getInt1 (true )});
1318- Value *MatchVal = Builder.CreateGEP (CharTy, MatchLCSSA, MatchCnt);
1339+ {MatchPredLCSSA, /* ZeroIsPoison=*/ Builder.getInt1 (true )}, nullptr ,
1340+ " match_idx" );
1341+ Value *MatchVal =
1342+ Builder.CreateGEP (CharTy, MatchLCSSA, MatchCnt, " match_res" );
13191343 Builder.CreateBr (ExitSucc);
13201344 DTU.applyUpdates ({{DominatorTree::Insert, BB3, ExitSucc}});
13211345
13221346 // (4) Check if we've reached the end of the needle array.
13231347 Builder.SetInsertPoint (BB4);
1324- Value *NextNeedle = Builder.CreateGEP (CharTy, Needle, ConstVF);
1348+ Value *NextNeedle =
1349+ Builder.CreateGEP (CharTy, Needle, ConstVF, " needle_next_vec" );
13251350 Builder.CreateCondBr (Builder.CreateICmpULT (NextNeedle, NeedleEnd), BB2, BB5);
13261351 DTU.applyUpdates (
13271352 {{DominatorTree::Insert, BB4, BB2}, {DominatorTree::Insert, BB4, BB5}});
13281353
13291354 // (5) Check if we've reached the end of the search array.
13301355 Builder.SetInsertPoint (BB5);
1331- Value *NextSearch = Builder.CreateGEP (CharTy, Search, ConstVF);
1356+ Value *NextSearch =
1357+ Builder.CreateGEP (CharTy, Search, ConstVF, " search_next_vec" );
13321358 Builder.CreateCondBr (Builder.CreateICmpULT (NextSearch, SearchEnd), BB1,
13331359 ExitFail);
13341360 DTU.applyUpdates ({{DominatorTree::Insert, BB5, BB1},
0 commit comments