diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index cfe1f1e9d7528..e3a4c11e63c88 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -965,6 +965,11 @@ class VPIRMetadata { /// Intersect this VPIRMetada object with \p MD, keeping only metadata /// nodes that are common to both. void intersect(const VPIRMetadata &MD); + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + /// Print metadata with node IDs. + void print(raw_ostream &O, const Module *M) const; +#endif }; /// This is a concrete Recipe that models a single VPlan-level instruction. @@ -4280,6 +4285,11 @@ class VPlan { /// Return the VPIRBasicBlock wrapping the header of the scalar loop. VPIRBasicBlock *getScalarHeader() const { return ScalarHeader; } + /// Return the Module from the scalar header. + const Module *getModule() const { + return ScalarHeader->getIRBasicBlock()->getModule(); + } + /// Return an ArrayRef containing VPIRBasicBlocks wrapping the exit blocks of /// the original scalar loop. ArrayRef getExitBlocks() const { return ExitBlocks; } diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 1ee405a62aa68..e755f498dab2d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -1449,6 +1449,7 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent, printFlags(O); printOperands(O, SlotTracker); + VPIRMetadata::print(O, getParent()->getPlan()->getModule()); if (auto DL = getDebugLoc()) { O << ", !dbg "; @@ -1675,6 +1676,27 @@ void VPIRMetadata::intersect(const VPIRMetadata &Other) { Metadata = std::move(MetadataIntersection); } +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +void VPIRMetadata::print(raw_ostream &O, const Module *M) const { + if (Metadata.empty()) + return; + + SmallVector MDNames; + M->getContext().getMDKindNames(MDNames); + + O << " ("; + interleaveComma(Metadata, O, [&](const auto &KindNodePair) { + auto [Kind, Node] = KindNodePair; + assert(Kind != 0 && "Debug metadata should not be managed by VPIRMetadata"); + assert(Kind < MDNames.size() && !MDNames[Kind].empty() && + "Unexpected unnamed metadata kind"); + O << "!" << MDNames[Kind] << " "; + Node->printAsOperand(O, M); + }); + O << ")"; +} +#endif + void VPWidenCallRecipe::execute(VPTransformState &State) { assert(State.VF.isVector() && "not widening"); assert(Variant != nullptr && "Can't create vector function."); @@ -1735,6 +1757,7 @@ void VPWidenCallRecipe::print(raw_ostream &O, const Twine &Indent, Op->printAsOperand(O, SlotTracker); }); O << ")"; + VPIRMetadata::print(O, getParent()->getPlan()->getModule()); O << " (using library function"; if (Variant->hasName()) @@ -1869,6 +1892,7 @@ void VPWidenIntrinsicRecipe::print(raw_ostream &O, const Twine &Indent, Op->printAsOperand(O, SlotTracker); }); O << ")"; + VPIRMetadata::print(O, getParent()->getPlan()->getModule()); } #endif @@ -1969,6 +1993,8 @@ void VPWidenSelectRecipe::print(raw_ostream &O, const Twine &Indent, O << ", "; getOperand(2)->printAsOperand(O, SlotTracker); O << (isInvariantCond() ? " (condition is loop invariant)" : ""); + + VPIRMetadata::print(O, getParent()->getPlan()->getModule()); } #endif @@ -2261,6 +2287,7 @@ void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent, O << " = " << Instruction::getOpcodeName(Opcode); printFlags(O); printOperands(O, SlotTracker); + VPIRMetadata::print(O, getParent()->getPlan()->getModule()); } #endif @@ -2342,6 +2369,7 @@ void VPWidenCastRecipe::print(raw_ostream &O, const Twine &Indent, printFlags(O); printOperands(O, SlotTracker); O << " to " << *getResultType(); + VPIRMetadata::print(O, getParent()->getPlan()->getModule()); } #endif @@ -3437,6 +3465,8 @@ void VPReplicateRecipe::print(raw_ostream &O, const Twine &Indent, if (shouldPack()) O << " (S->V)"; + + VPIRMetadata::print(O, getParent()->getPlan()->getModule()); } #endif @@ -3623,6 +3653,7 @@ void VPWidenLoadRecipe::print(raw_ostream &O, const Twine &Indent, printAsOperand(O, SlotTracker); O << " = load "; printOperands(O, SlotTracker); + VPIRMetadata::print(O, getParent()->getPlan()->getModule()); } #endif @@ -3702,6 +3733,7 @@ void VPWidenLoadEVLRecipe::print(raw_ostream &O, const Twine &Indent, printAsOperand(O, SlotTracker); O << " = vp.load "; printOperands(O, SlotTracker); + VPIRMetadata::print(O, getParent()->getPlan()->getModule()); } #endif @@ -3744,6 +3776,7 @@ void VPWidenStoreRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { O << Indent << "WIDEN store "; printOperands(O, SlotTracker); + VPIRMetadata::print(O, getParent()->getPlan()->getModule()); } #endif @@ -3809,6 +3842,7 @@ void VPWidenStoreEVLRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { O << Indent << "WIDEN vp.store "; printOperands(O, SlotTracker); + VPIRMetadata::print(O, getParent()->getPlan()->getModule()); } #endif @@ -4097,6 +4131,8 @@ void VPInterleaveRecipe::print(raw_ostream &O, const Twine &Indent, } ++OpIdx; } + + VPIRMetadata::print(O, getParent()->getPlan()->getModule()); } #endif @@ -4241,6 +4277,8 @@ void VPInterleaveEVLRecipe::print(raw_ostream &O, const Twine &Indent, } ++OpIdx; } + + VPIRMetadata::print(O, getParent()->getPlan()->getModule()); } #endif diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-metadata.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-metadata.ll index 857b9131a0b8c..fb49e94ee67bc 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing-metadata.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-metadata.ll @@ -7,11 +7,11 @@ define void @test_widen_metadata(ptr noalias %A, ptr noalias %B, i32 %n) { ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { ; CHECK: vector loop: { ; CHECK: vector.body: -; CHECK: WIDEN ir<%lv> = load vp<{{.*}}> -; CHECK: WIDEN-CAST ir<%conv> = sitofp ir<%lv> to float -; CHECK: WIDEN ir<%mul> = fmul ir<%conv>, ir<2.000000e+00> +; CHECK: WIDEN ir<%lv> = load vp<{{.*}}> (!tbaa !{{[0-9]+}}) +; CHECK: WIDEN-CAST ir<%conv> = sitofp ir<%lv> to float (!fpmath !{{[0-9]+}}) +; CHECK: WIDEN ir<%mul> = fmul ir<%conv>, ir<2.000000e+00> (!fpmath !{{[0-9]+}}) ; CHECK: WIDEN-CAST ir<%conv.back> = fptosi ir<%mul> to i32 -; CHECK: WIDEN store vp<{{.*}}>, ir<%conv.back> +; CHECK: WIDEN store vp<{{.*}}>, ir<%conv.back> (!tbaa !{{[0-9]+}}) ; entry: br label %loop @@ -40,9 +40,9 @@ define void @test_intrinsic_with_metadata(ptr noalias %A, ptr noalias %B, i32 %n ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { ; CHECK: vector loop: { ; CHECK: vector.body: -; CHECK: WIDEN ir<%lv> = load vp<{{.*}}> -; CHECK: WIDEN-INTRINSIC ir<%sqrt> = call llvm.sqrt(ir<%lv>) -; CHECK: WIDEN store vp<{{.*}}>, ir<%sqrt> +; CHECK: WIDEN ir<%lv> = load vp<{{.*}}> (!tbaa !{{[0-9]+}}) +; CHECK: WIDEN-INTRINSIC ir<%sqrt> = call llvm.sqrt(ir<%lv>) (!fpmath !{{[0-9]+}}) +; CHECK: WIDEN store vp<{{.*}}>, ir<%sqrt> (!tbaa !{{[0-9]+}}) ; entry: br label %loop @@ -67,11 +67,11 @@ define void @test_widen_with_multiple_metadata(ptr noalias %A, ptr noalias %B, i ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { ; CHECK: vector loop: { ; CHECK: vector.body: -; CHECK: WIDEN ir<%lv> = load vp<{{.*}}> +; CHECK: WIDEN ir<%lv> = load vp<{{.*}}> (!tbaa !{{[0-9]+}}) ; CHECK: WIDEN-CAST ir<%conv> = sitofp ir<%lv> to float ; CHECK: WIDEN ir<%mul> = fmul ir<%conv>, ir<2.000000e+00> ; CHECK: WIDEN-CAST ir<%conv.back> = fptosi ir<%mul> to i32 -; CHECK: WIDEN store vp<{{.*}}>, ir<%conv.back> +; CHECK: WIDEN store vp<{{.*}}>, ir<%conv.back> (!tbaa !{{[0-9]+}}) ; entry: br label %loop diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp index 59a9ea1a720b3..49fedc83ce9b0 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -731,18 +731,18 @@ TEST_F(VPBasicBlockTest, print) { VPBB2->setName("bb2"); VPBlockUtils::connectBlocks(VPBB1, VPBB2); + VPBlockUtils::connectBlocks(VPBB2, Plan.getScalarHeader()); + VPBlockUtils::connectBlocks(VPBB0, VPBB1); - // Check printing an instruction without associated VPlan. + // Check printing an instruction with associated VPlan. { std::string I3Dump; raw_string_ostream OS(I3Dump); - VPSlotTracker SlotTracker; + VPSlotTracker SlotTracker(&Plan); I3->print(OS, "", SlotTracker); - EXPECT_EQ("EMIT br , ", I3Dump); + EXPECT_EQ("EMIT br vp<%2>, vp<%3>", I3Dump); } - VPBlockUtils::connectBlocks(VPBB2, Plan.getScalarHeader()); - VPBlockUtils::connectBlocks(VPBB0, VPBB1); std::string FullDump; raw_string_ostream OS(FullDump); Plan.printDOT(OS); @@ -1509,12 +1509,18 @@ TEST_F(VPRecipeTest, dumpRecipeUnnamedVPValuesNotInPlanOrBlock) { auto *AI = BinaryOperator::CreateAdd(PoisonValue::get(Int32), PoisonValue::get(Int32)); AI->setName("a"); - VPValue *ExtVPV1 = getPlan().getOrAddLiveIn(ConstantInt::get(Int32, 1)); - VPValue *ExtVPV2 = getPlan().getOrAddLiveIn(AI); + VPlan &Plan = getPlan(); + VPValue *ExtVPV1 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1)); + VPValue *ExtVPV2 = Plan.getOrAddLiveIn(AI); VPInstruction *I1 = new VPInstruction(Instruction::Add, {ExtVPV1, ExtVPV2}); VPInstruction *I2 = new VPInstruction(Instruction::Mul, {I1, I1}); + // Add instructions to a block in the plan so they have access to Module + VPBasicBlock *VPBB = Plan.getEntry(); + VPBB->appendRecipe(I1); + VPBB->appendRecipe(I2); + // Check printing I1. { // Use EXPECT_EXIT to capture stderr and compare against expected output. @@ -1526,7 +1532,7 @@ TEST_F(VPRecipeTest, dumpRecipeUnnamedVPValuesNotInPlanOrBlock) { VPV->dump(); exit(0); }, - testing::ExitedWithCode(0), "EMIT = add ir<1>, ir<%a>"); + testing::ExitedWithCode(0), "EMIT vp<%1> = add ir<1>, ir<%a>"); // Test VPRecipeBase::dump(). VPRecipeBase *R = I1; @@ -1535,7 +1541,7 @@ TEST_F(VPRecipeTest, dumpRecipeUnnamedVPValuesNotInPlanOrBlock) { R->dump(); exit(0); }, - testing::ExitedWithCode(0), "EMIT = add ir<1>, ir<%a>"); + testing::ExitedWithCode(0), "EMIT vp<%1> = add ir<1>, ir<%a>"); // Test VPDef::dump(). VPDef *D = I1; @@ -1544,7 +1550,7 @@ TEST_F(VPRecipeTest, dumpRecipeUnnamedVPValuesNotInPlanOrBlock) { D->dump(); exit(0); }, - testing::ExitedWithCode(0), "EMIT = add ir<1>, ir<%a>"); + testing::ExitedWithCode(0), "EMIT vp<%1> = add ir<1>, ir<%a>"); } // Check printing I2. { @@ -1557,7 +1563,7 @@ TEST_F(VPRecipeTest, dumpRecipeUnnamedVPValuesNotInPlanOrBlock) { VPV->dump(); exit(0); }, - testing::ExitedWithCode(0), "EMIT = mul , "); + testing::ExitedWithCode(0), "EMIT vp<%2> = mul vp<%1>, vp<%1>"); // Test VPRecipeBase::dump(). VPRecipeBase *R = I2; @@ -1566,7 +1572,7 @@ TEST_F(VPRecipeTest, dumpRecipeUnnamedVPValuesNotInPlanOrBlock) { R->dump(); exit(0); }, - testing::ExitedWithCode(0), "EMIT = mul , "); + testing::ExitedWithCode(0), "EMIT vp<%2> = mul vp<%1>, vp<%1>"); // Test VPDef::dump(). VPDef *D = I2; @@ -1575,11 +1581,9 @@ TEST_F(VPRecipeTest, dumpRecipeUnnamedVPValuesNotInPlanOrBlock) { D->dump(); exit(0); }, - testing::ExitedWithCode(0), "EMIT = mul , "); + testing::ExitedWithCode(0), "EMIT vp<%2> = mul vp<%1>, vp<%1>"); } - delete I2; - delete I1; delete AI; } diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h index ed6e13b4add3d..f763b3465de36 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h +++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h @@ -86,15 +86,22 @@ class VPlanTestIRBase : public testing::Test { class VPlanTestBase : public testing::Test { protected: LLVMContext C; - std::unique_ptr ScalarHeader; + std::unique_ptr M; + Function *F; + BasicBlock *ScalarHeader; SmallVector> Plans; - VPlanTestBase() : ScalarHeader(BasicBlock::Create(C, "scalar.header")) { - BranchInst::Create(&*ScalarHeader, &*ScalarHeader); + VPlanTestBase() { + M = std::make_unique("VPlanTest", C); + FunctionType *FTy = FunctionType::get(Type::getVoidTy(C), false); + F = Function::Create(FTy, GlobalValue::ExternalLinkage, "test_function", + M.get()); + ScalarHeader = BasicBlock::Create(C, "scalar.header", F); + BranchInst::Create(ScalarHeader, ScalarHeader); } VPlan &getPlan(VPValue *TC = nullptr) { - Plans.push_back(std::make_unique(&*ScalarHeader, TC)); + Plans.push_back(std::make_unique(ScalarHeader, TC)); return *Plans.back(); } };