diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h index 04ea769bd06d1..d8f26aba81d78 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolution.h +++ b/llvm/include/llvm/Analysis/ScalarEvolution.h @@ -563,8 +563,13 @@ class ScalarEvolution { LLVM_ABI const SCEV *getConstant(ConstantInt *V); LLVM_ABI const SCEV *getConstant(const APInt &Val); LLVM_ABI const SCEV *getConstant(Type *Ty, uint64_t V, bool isSigned = false); - LLVM_ABI const SCEV *getLosslessPtrToIntExpr(const SCEV *Op, - unsigned Depth = 0); + LLVM_ABI const SCEV *getLosslessPtrToIntOrAddrExpr(SCEVTypes Kind, + const SCEV *Op, + unsigned Depth = 0); + LLVM_ABI const SCEV *getLosslessPtrToIntExpr(const SCEV *Op); + LLVM_ABI const SCEV *getLosslessPtrToAddrExpr(const SCEV *Op); + + LLVM_ABI const SCEV *getPtrToAddrExpr(const SCEV *Op); LLVM_ABI const SCEV *getPtrToIntExpr(const SCEV *Op, Type *Ty); LLVM_ABI const SCEV *getTruncateExpr(const SCEV *Op, Type *Ty, unsigned Depth = 0); diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionDivision.h b/llvm/include/llvm/Analysis/ScalarEvolutionDivision.h index 7c78487fea8c0..d3df7e346b4a5 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolutionDivision.h +++ b/llvm/include/llvm/Analysis/ScalarEvolutionDivision.h @@ -46,6 +46,7 @@ struct SCEVDivision : public SCEVVisitor { // Except in the trivial case described above, we do not know how to divide // Expr by Denominator for the following functions with empty implementation. + void visitPtrToAddrExpr(const SCEVPtrToAddrExpr *Numerator) {} void visitPtrToIntExpr(const SCEVPtrToIntExpr *Numerator) {} void visitTruncateExpr(const SCEVTruncateExpr *Numerator) {} void visitZeroExtendExpr(const SCEVZeroExtendExpr *Numerator) {} diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h index 13b9e1b812942..788c9559f8572 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h +++ b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h @@ -53,6 +53,7 @@ enum SCEVTypes : unsigned short { scSMinExpr, scSequentialUMinExpr, scPtrToInt, + scPtrToAddr, scUnknown, scCouldNotCompute }; @@ -121,8 +122,9 @@ class SCEVCastExpr : public SCEV { /// Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const SCEV *S) { - return S->getSCEVType() == scPtrToInt || S->getSCEVType() == scTruncate || - S->getSCEVType() == scZeroExtend || S->getSCEVType() == scSignExtend; + return S->getSCEVType() == scPtrToAddr || S->getSCEVType() == scPtrToInt || + S->getSCEVType() == scTruncate || S->getSCEVType() == scZeroExtend || + S->getSCEVType() == scSignExtend; } }; @@ -138,6 +140,18 @@ class SCEVPtrToIntExpr : public SCEVCastExpr { static bool classof(const SCEV *S) { return S->getSCEVType() == scPtrToInt; } }; +/// This class represents a cast from a pointer to a pointer-sized integer +/// value, without capturing the provenance of the pointer. +class SCEVPtrToAddrExpr : public SCEVCastExpr { + friend class ScalarEvolution; + + SCEVPtrToAddrExpr(const FoldingSetNodeIDRef ID, const SCEV *Op, Type *ITy); + +public: + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { return S->getSCEVType() == scPtrToAddr; } +}; + /// This is the base class for unary integral cast operator classes. class SCEVIntegralCastExpr : public SCEVCastExpr { protected: @@ -615,6 +629,8 @@ template struct SCEVVisitor { return ((SC *)this)->visitConstant((const SCEVConstant *)S); case scVScale: return ((SC *)this)->visitVScale((const SCEVVScale *)S); + case scPtrToAddr: + return ((SC *)this)->visitPtrToAddrExpr((const SCEVPtrToAddrExpr *)S); case scPtrToInt: return ((SC *)this)->visitPtrToIntExpr((const SCEVPtrToIntExpr *)S); case scTruncate: @@ -685,6 +701,7 @@ template class SCEVTraversal { case scVScale: case scUnknown: continue; + case scPtrToAddr: case scPtrToInt: case scTruncate: case scZeroExtend: @@ -774,6 +791,11 @@ class SCEVRewriteVisitor : public SCEVVisitor { const SCEV *visitVScale(const SCEVVScale *VScale) { return VScale; } + const SCEV *visitPtrToAddrExpr(const SCEVPtrToAddrExpr *Expr) { + const SCEV *Operand = ((SC *)this)->visit(Expr->getOperand()); + return Operand == Expr->getOperand() ? Expr : SE.getPtrToAddrExpr(Operand); + } + const SCEV *visitPtrToIntExpr(const SCEVPtrToIntExpr *Expr) { const SCEV *Operand = ((SC *)this)->visit(Expr->getOperand()); return Operand == Expr->getOperand() diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h index 310118078695c..c986f78db5c19 100644 --- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h +++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h @@ -498,6 +498,8 @@ class SCEVExpander : public SCEVVisitor { Value *visitVScale(const SCEVVScale *S); + Value *visitPtrToAddrExpr(const SCEVPtrToAddrExpr *S); + Value *visitPtrToIntExpr(const SCEVPtrToIntExpr *S); Value *visitTruncateExpr(const SCEVTruncateExpr *S); diff --git a/llvm/lib/Analysis/DependenceAnalysis.cpp b/llvm/lib/Analysis/DependenceAnalysis.cpp index 84ee8c0bf3e18..de3a585e864a0 100644 --- a/llvm/lib/Analysis/DependenceAnalysis.cpp +++ b/llvm/lib/Analysis/DependenceAnalysis.cpp @@ -354,6 +354,9 @@ struct SCEVMonotonicityChecker SCEVMonotonicity visitMulExpr(const SCEVMulExpr *Expr) { return invariantOrUnknown(Expr); } + SCEVMonotonicity visitPtrToAddrExpr(const SCEVPtrToAddrExpr *Expr) { + return invariantOrUnknown(Expr); + } SCEVMonotonicity visitPtrToIntExpr(const SCEVPtrToIntExpr *Expr) { return invariantOrUnknown(Expr); } diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 7597f3ad685a0..a160c4c071cca 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -277,11 +277,13 @@ void SCEV::print(raw_ostream &OS) const { case scVScale: OS << "vscale"; return; + case scPtrToAddr: case scPtrToInt: { - const SCEVPtrToIntExpr *PtrToInt = cast(this); - const SCEV *Op = PtrToInt->getOperand(); - OS << "(ptrtoint " << *Op->getType() << " " << *Op << " to " - << *PtrToInt->getType() << ")"; + const SCEVCastExpr *PtrCast = cast(this); + const SCEV *Op = PtrCast->getOperand(); + StringRef OpS = getSCEVType() == scPtrToAddr ? "addr" : "int"; + OS << "(ptrto" << OpS << " " << *Op->getType() << " " << *Op << " to " + << *PtrCast->getType() << ")"; return; } case scTruncate: { @@ -386,6 +388,7 @@ Type *SCEV::getType() const { return cast(this)->getType(); case scVScale: return cast(this)->getType(); + case scPtrToAddr: case scPtrToInt: case scTruncate: case scZeroExtend: @@ -420,6 +423,7 @@ ArrayRef SCEV::operands() const { case scVScale: case scUnknown: return {}; + case scPtrToAddr: case scPtrToInt: case scTruncate: case scZeroExtend: @@ -512,6 +516,13 @@ SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy, const SCEV *op, Type *ty) : SCEV(ID, SCEVTy, computeExpressionSize(op)), Op(op), Ty(ty) {} +SCEVPtrToAddrExpr::SCEVPtrToAddrExpr(const FoldingSetNodeIDRef ID, + const SCEV *Op, Type *ITy) + : SCEVCastExpr(ID, scPtrToAddr, Op, ITy) { + assert(getOperand()->getType()->isPointerTy() && Ty->isIntegerTy() && + "Must be a non-bit-width-changing pointer-to-integer cast!"); +} + SCEVPtrToIntExpr::SCEVPtrToIntExpr(const FoldingSetNodeIDRef ID, const SCEV *Op, Type *ITy) : SCEVCastExpr(ID, scPtrToInt, Op, ITy) { @@ -724,6 +735,7 @@ CompareSCEVComplexity(const LoopInfo *const LI, const SCEV *LHS, case scTruncate: case scZeroExtend: case scSignExtend: + case scPtrToAddr: case scPtrToInt: case scAddExpr: case scMulExpr: @@ -1004,10 +1016,11 @@ SCEVAddRecExpr::evaluateAtIteration(ArrayRef Operands, // SCEV Expression folder implementations //===----------------------------------------------------------------------===// -const SCEV *ScalarEvolution::getLosslessPtrToIntExpr(const SCEV *Op, - unsigned Depth) { +const SCEV *ScalarEvolution::getLosslessPtrToIntOrAddrExpr(SCEVTypes Kind, + const SCEV *Op, + unsigned Depth) { assert(Depth <= 1 && - "getLosslessPtrToIntExpr() should self-recurse at most once."); + "getLosslessPtrToIntOrAddrExpr() should self-recurse at most once."); // We could be called with an integer-typed operands during SCEV rewrites. // Since the operand is an integer already, just perform zext/trunc/self cast. @@ -1052,35 +1065,45 @@ const SCEV *ScalarEvolution::getLosslessPtrToIntExpr(const SCEV *Op, // Create an explicit cast node. // We can reuse the existing insert position since if we get here, // we won't have made any changes which would invalidate it. - SCEV *S = new (SCEVAllocator) - SCEVPtrToIntExpr(ID.Intern(SCEVAllocator), Op, IntPtrTy); + SCEV *S; + if (Kind == scPtrToInt) { + S = new (SCEVAllocator) + SCEVPtrToIntExpr(ID.Intern(SCEVAllocator), Op, IntPtrTy); + } else { + S = new (SCEVAllocator) + SCEVPtrToAddrExpr(ID.Intern(SCEVAllocator), Op, IntPtrTy); + } UniqueSCEVs.InsertNode(S, IP); registerUser(S, Op); return S; } - assert(Depth == 0 && "getLosslessPtrToIntExpr() should not self-recurse for " - "non-SCEVUnknown's."); + assert(Depth == 0 && + "getLosslessPtrToIntOrAddrExpr() should not self-recurse for " + "non-SCEVUnknown's."); // Otherwise, we've got some expression that is more complex than just a - // single SCEVUnknown. But we don't want to have a SCEVPtrToIntExpr of an - // arbitrary expression, we want to have SCEVPtrToIntExpr of an SCEVUnknown - // only, and the expressions must otherwise be integer-typed. - // So sink the cast down to the SCEVUnknown's. + // single SCEVUnknown. But we don't want to have a SCEVPtrTo(Int|Addr)Expr of + // an arbitrary expression, we want to have SCEVPtrTo(Int|Addr)Expr of an + // SCEVUnknown only, and the expressions must otherwise be integer-typed. So + // sink the cast down to the SCEVUnknown's. - /// The SCEVPtrToIntSinkingRewriter takes a scalar evolution expression, + /// The SCEVPtrToIntOrAddrSinkingRewriter takes a scalar evolution expression, /// which computes a pointer-typed value, and rewrites the whole expression /// tree so that *all* the computations are done on integers, and the only /// pointer-typed operands in the expression are SCEVUnknown. - class SCEVPtrToIntSinkingRewriter - : public SCEVRewriteVisitor { - using Base = SCEVRewriteVisitor; + class SCEVPtrToIntOrAddrSinkingRewriter + : public SCEVRewriteVisitor { + using Base = SCEVRewriteVisitor; + const SCEVTypes Kind; public: - SCEVPtrToIntSinkingRewriter(ScalarEvolution &SE) : SCEVRewriteVisitor(SE) {} + SCEVPtrToIntOrAddrSinkingRewriter(SCEVTypes Kind, ScalarEvolution &SE) + : SCEVRewriteVisitor(SE), Kind(Kind) {} - static const SCEV *rewrite(const SCEV *Scev, ScalarEvolution &SE) { - SCEVPtrToIntSinkingRewriter Rewriter(SE); + static const SCEV *rewrite(const SCEV *Scev, SCEVTypes Kind, + ScalarEvolution &SE) { + SCEVPtrToIntOrAddrSinkingRewriter Rewriter(Kind, SE); return Rewriter.visit(Scev); } @@ -1116,18 +1139,31 @@ const SCEV *ScalarEvolution::getLosslessPtrToIntExpr(const SCEV *Op, const SCEV *visitUnknown(const SCEVUnknown *Expr) { assert(Expr->getType()->isPointerTy() && "Should only reach pointer-typed SCEVUnknown's."); - return SE.getLosslessPtrToIntExpr(Expr, /*Depth=*/1); + return SE.getLosslessPtrToIntOrAddrExpr(Kind, Expr, /*Depth=*/1); } }; // And actually perform the cast sinking. - const SCEV *IntOp = SCEVPtrToIntSinkingRewriter::rewrite(Op, *this); + const SCEV *IntOp = + SCEVPtrToIntOrAddrSinkingRewriter::rewrite(Op, Kind, *this); assert(IntOp->getType()->isIntegerTy() && "We must have succeeded in sinking the cast, " "and ending up with an integer-typed expression!"); return IntOp; } +const SCEV *ScalarEvolution::getLosslessPtrToAddrExpr(const SCEV *Op) { + return getLosslessPtrToIntOrAddrExpr(scPtrToAddr, Op); +} + +const SCEV *ScalarEvolution::getLosslessPtrToIntExpr(const SCEV *Op) { + return getLosslessPtrToIntOrAddrExpr(scPtrToInt, Op); +} + +const SCEV *ScalarEvolution::getPtrToAddrExpr(const SCEV *Op) { + return getLosslessPtrToAddrExpr(Op); +} + const SCEV *ScalarEvolution::getPtrToIntExpr(const SCEV *Op, Type *Ty) { assert(Ty->isIntegerTy() && "Target type must be an integer type!"); @@ -4072,6 +4108,8 @@ class SCEVSequentialMinMaxDeduplicatingVisitor final RetVal visitVScale(const SCEVVScale *VScale) { return VScale; } + RetVal visitPtrToAddrExpr(const SCEVPtrToAddrExpr *Expr) { return Expr; } + RetVal visitPtrToIntExpr(const SCEVPtrToIntExpr *Expr) { return Expr; } RetVal visitTruncateExpr(const SCEVTruncateExpr *Expr) { return Expr; } @@ -4122,6 +4160,7 @@ static bool scevUnconditionallyPropagatesPoisonFromOperands(SCEVTypes Kind) { case scTruncate: case scZeroExtend: case scSignExtend: + case scPtrToAddr: case scPtrToInt: case scAddExpr: case scMulExpr: @@ -6351,8 +6390,9 @@ APInt ScalarEvolution::getConstantMultipleImpl(const SCEV *S, switch (S->getSCEVType()) { case scConstant: return cast(S)->getAPInt(); + case scPtrToAddr: case scPtrToInt: - return getConstantMultiple(cast(S)->getOperand(), CtxI); + return getConstantMultiple(cast(S)->getOperand()); case scUDivExpr: case scVScale: return APInt(BitWidth, 1); @@ -6629,6 +6669,7 @@ ScalarEvolution::getRangeRefIter(const SCEV *S, case scTruncate: case scZeroExtend: case scSignExtend: + case scPtrToAddr: case scPtrToInt: case scAddExpr: case scMulExpr: @@ -6756,10 +6797,11 @@ const ConstantRange &ScalarEvolution::getRangeRef( SExt, SignHint, ConservativeResult.intersectWith(X.signExtend(BitWidth), RangeType)); } + case scPtrToAddr: case scPtrToInt: { - const SCEVPtrToIntExpr *PtrToInt = cast(S); - ConstantRange X = getRangeRef(PtrToInt->getOperand(), SignHint, Depth + 1); - return setRange(PtrToInt, SignHint, X); + const SCEVCastExpr *Cast = cast(S); + ConstantRange X = getRangeRef(Cast->getOperand(), SignHint, Depth + 1); + return setRange(Cast, SignHint, X); } case scAddExpr: { const SCEVAddExpr *Add = cast(S); @@ -7652,6 +7694,7 @@ ScalarEvolution::getOperandsToCreate(Value *V, SmallVectorImpl &Ops) { case Instruction::Trunc: case Instruction::ZExt: case Instruction::SExt: + case Instruction::PtrToAddr: case Instruction::PtrToInt: Ops.push_back(U->getOperand(0)); return nullptr; @@ -8124,13 +8167,16 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) { return getSCEV(U->getOperand(0)); break; + case Instruction::PtrToAddr: case Instruction::PtrToInt: { // Pointer to integer cast is straight-forward, so do model it. const SCEV *Op = getSCEV(U->getOperand(0)); Type *DstIntTy = U->getType(); // But only if effective SCEV (integer) type is wide enough to represent // all possible pointer values. - const SCEV *IntOp = getPtrToIntExpr(Op, DstIntTy); + const SCEV *IntOp = U->getOpcode() == Instruction::PtrToInt + ? getPtrToIntExpr(Op, DstIntTy) + : getPtrToAddrExpr(Op); if (isa(IntOp)) return getUnknown(V); return IntOp; @@ -9932,6 +9978,13 @@ static Constant *BuildConstantFromSCEV(const SCEV *V) { return cast(V)->getValue(); case scUnknown: return dyn_cast(cast(V)->getValue()); + case scPtrToAddr: { + const SCEVPtrToAddrExpr *P2I = cast(V); + if (Constant *CastOp = BuildConstantFromSCEV(P2I->getOperand())) + return ConstantExpr::getPtrToAddr(CastOp, P2I->getType()); + + return nullptr; + } case scPtrToInt: { const SCEVPtrToIntExpr *P2I = cast(V); if (Constant *CastOp = BuildConstantFromSCEV(P2I->getOperand())) @@ -9990,6 +10043,7 @@ ScalarEvolution::getWithOperands(const SCEV *S, case scTruncate: case scZeroExtend: case scSignExtend: + case scPtrToAddr: case scPtrToInt: return getCastExpr(S->getSCEVType(), NewOps[0], S->getType()); case scAddRecExpr: { @@ -10074,6 +10128,7 @@ const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) { case scTruncate: case scZeroExtend: case scSignExtend: + case scPtrToAddr: case scPtrToInt: case scAddExpr: case scMulExpr: @@ -14169,6 +14224,7 @@ ScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L) { case scTruncate: case scZeroExtend: case scSignExtend: + case scPtrToAddr: case scPtrToInt: case scAddExpr: case scMulExpr: @@ -14250,6 +14306,7 @@ ScalarEvolution::computeBlockDisposition(const SCEV *S, const BasicBlock *BB) { case scTruncate: case scZeroExtend: case scSignExtend: + case scPtrToAddr: case scPtrToInt: case scAddExpr: case scMulExpr: diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp index 9035e58a707c4..6f08eabdd90de 100644 --- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp +++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp @@ -457,6 +457,7 @@ const Loop *SCEVExpander::getRelevantLoop(const SCEV *S) { case scTruncate: case scZeroExtend: case scSignExtend: + case scPtrToAddr: case scPtrToInt: case scAddExpr: case scMulExpr: @@ -1433,6 +1434,12 @@ Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) { return expand(T); } +Value *SCEVExpander::visitPtrToAddrExpr(const SCEVPtrToAddrExpr *S) { + Value *V = expand(S->getOperand()); + return ReuseOrCreateCast(V, S->getType(), CastInst::PtrToAddr, + GetOptimalInsertionPointForCastOf(V)); +} + Value *SCEVExpander::visitPtrToIntExpr(const SCEVPtrToIntExpr *S) { Value *V = expand(S->getOperand()); return ReuseOrCreateCast(V, S->getType(), CastInst::PtrToInt, @@ -1957,6 +1964,9 @@ template static InstructionCost costAndCollectOperands( case scConstant: case scVScale: return 0; + case scPtrToAddr: + Cost = CastCost(Instruction::PtrToAddr); + break; case scPtrToInt: Cost = CastCost(Instruction::PtrToInt); break; @@ -2080,6 +2090,7 @@ bool SCEVExpander::isHighCostExpansionHelper( return Cost > Budget; } case scTruncate: + case scPtrToAddr: case scPtrToInt: case scZeroExtend: case scSignExtend: { diff --git a/llvm/test/Analysis/ScalarEvolution/ptrtoaddr-i32-index-width.ll b/llvm/test/Analysis/ScalarEvolution/ptrtoaddr-i32-index-width.ll new file mode 100644 index 0000000000000..88a78a1c73d77 --- /dev/null +++ b/llvm/test/Analysis/ScalarEvolution/ptrtoaddr-i32-index-width.ll @@ -0,0 +1,70 @@ +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py +; RUN: opt -passes='print' -disable-output %s 2>&1 | FileCheck %s + +target datalayout="p:64:64:64:32" + +define void @ptrtoaddr(ptr %in, ptr %out0) { +; CHECK-LABEL: 'ptrtoaddr' +; CHECK-NEXT: Classifying expressions for: @ptrtoaddr +; CHECK-NEXT: %p0 = ptrtoaddr ptr %in to i32 +; CHECK-NEXT: --> %p0 U: full-set S: full-set +; CHECK-NEXT: Determining loop execution counts for: @ptrtoaddr +; + %p0 = ptrtoaddr ptr %in to i32 + store i32 %p0, ptr %out0 + ret void +} + +define void @ptrtoaddr_as1(ptr addrspace(1) %in, ptr %out0) { +; CHECK-LABEL: 'ptrtoaddr_as1' +; CHECK-NEXT: Classifying expressions for: @ptrtoaddr_as1 +; CHECK-NEXT: %p0 = ptrtoaddr ptr addrspace(1) %in to i32 +; CHECK-NEXT: --> %p0 U: full-set S: full-set +; CHECK-NEXT: Determining loop execution counts for: @ptrtoaddr_as1 +; + %p0 = ptrtoaddr ptr addrspace(1) %in to i32 + store i32 %p0, ptr %out0 + ret void +} + +define void @ptrtoaddr_of_bitcast(ptr %in, ptr %out0) { +; CHECK-LABEL: 'ptrtoaddr_of_bitcast' +; CHECK-NEXT: Classifying expressions for: @ptrtoaddr_of_bitcast +; CHECK-NEXT: %in_casted = bitcast ptr %in to ptr +; CHECK-NEXT: --> %in U: full-set S: full-set +; CHECK-NEXT: %p0 = ptrtoaddr ptr %in_casted to i32 +; CHECK-NEXT: --> %p0 U: full-set S: full-set +; CHECK-NEXT: Determining loop execution counts for: @ptrtoaddr_of_bitcast +; + %in_casted = bitcast ptr %in to ptr + %p0 = ptrtoaddr ptr %in_casted to i32 + store i32 %p0, ptr %out0 + ret void +} + +define void @ptrtoaddr_of_nullptr(ptr %out0) { +; CHECK-LABEL: 'ptrtoaddr_of_nullptr' +; CHECK-NEXT: Classifying expressions for: @ptrtoaddr_of_nullptr +; CHECK-NEXT: %p0 = ptrtoaddr ptr null to i32 +; CHECK-NEXT: --> %p0 U: full-set S: full-set +; CHECK-NEXT: Determining loop execution counts for: @ptrtoaddr_of_nullptr +; + %p0 = ptrtoaddr ptr null to i32 + store i32 %p0, ptr %out0 + ret void +} + +define void @ptrtoaddr_of_gep(ptr %in, ptr %out0) { +; CHECK-LABEL: 'ptrtoaddr_of_gep' +; CHECK-NEXT: Classifying expressions for: @ptrtoaddr_of_gep +; CHECK-NEXT: %in_adj = getelementptr inbounds i8, ptr %in, i64 42 +; CHECK-NEXT: --> (42 + %in) U: full-set S: full-set +; CHECK-NEXT: %p0 = ptrtoaddr ptr %in_adj to i32 +; CHECK-NEXT: --> %p0 U: full-set S: full-set +; CHECK-NEXT: Determining loop execution counts for: @ptrtoaddr_of_gep +; + %in_adj = getelementptr inbounds i8, ptr %in, i64 42 + %p0 = ptrtoaddr ptr %in_adj to i32 + store i32 %p0, ptr %out0 + ret void +} diff --git a/llvm/test/Analysis/ScalarEvolution/ptrtoaddr.ll b/llvm/test/Analysis/ScalarEvolution/ptrtoaddr.ll index ebab9f0f20323..3710a00c043f0 100644 --- a/llvm/test/Analysis/ScalarEvolution/ptrtoaddr.ll +++ b/llvm/test/Analysis/ScalarEvolution/ptrtoaddr.ll @@ -1,97 +1,40 @@ ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py -; RUN: opt < %s --data-layout="e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" -S -disable-output -disable-verify "-passes=print" 2>&1 | FileCheck --check-prefixes=ALL,X64 %s -; RUN: opt < %s --data-layout="e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-f80:32-n8:16:32-S128" -S -disable-output -disable-verify "-passes=print" 2>&1 | FileCheck --check-prefixes=ALL,X32 %s +; RUN: opt -passes='print' -disable-output %s 2>&1 | FileCheck %s -declare void @useptr(ptr) +target datalayout="p64:64:64:64" define void @ptrtoaddr(ptr %in, ptr %out0, ptr %out1, ptr %out2, ptr %out3) { -; X64-LABEL: 'ptrtoaddr' -; X64-NEXT: Classifying expressions for: @ptrtoaddr -; X64-NEXT: %p0 = ptrtoaddr ptr %in to i64 -; X64-NEXT: --> %p0 U: full-set S: full-set -; X64-NEXT: %p1 = ptrtoaddr ptr %in to i32 -; X64-NEXT: --> %p1 U: full-set S: full-set -; X64-NEXT: %p2 = ptrtoaddr ptr %in to i16 -; X64-NEXT: --> %p2 U: full-set S: full-set -; X64-NEXT: %p3 = ptrtoaddr ptr %in to i128 -; X64-NEXT: --> %p3 U: full-set S: full-set -; X64-NEXT: Determining loop execution counts for: @ptrtoaddr -; -; X32-LABEL: 'ptrtoaddr' -; X32-NEXT: Classifying expressions for: @ptrtoaddr -; X32-NEXT: %p0 = ptrtoaddr ptr %in to i64 -; X32-NEXT: --> %p0 U: full-set S: full-set -; X32-NEXT: %p1 = ptrtoaddr ptr %in to i32 -; X32-NEXT: --> %p1 U: full-set S: full-set -; X32-NEXT: %p2 = ptrtoaddr ptr %in to i16 -; X32-NEXT: --> %p2 U: full-set S: full-set -; X32-NEXT: %p3 = ptrtoaddr ptr %in to i128 -; X32-NEXT: --> %p3 U: full-set S: full-set -; X32-NEXT: Determining loop execution counts for: @ptrtoaddr +; CHECK-LABEL: 'ptrtoaddr' +; CHECK-NEXT: Classifying expressions for: @ptrtoaddr +; CHECK-NEXT: %p0 = ptrtoaddr ptr %in to i64 +; CHECK-NEXT: --> (ptrtoaddr ptr %in to i64) U: full-set S: full-set +; CHECK-NEXT: Determining loop execution counts for: @ptrtoaddr ; %p0 = ptrtoaddr ptr %in to i64 - %p1 = ptrtoaddr ptr %in to i32 - %p2 = ptrtoaddr ptr %in to i16 - %p3 = ptrtoaddr ptr %in to i128 store i64 %p0, ptr %out0 - store i32 %p1, ptr %out1 - store i16 %p2, ptr %out2 - store i128 %p3, ptr %out3 ret void } define void @ptrtoaddr_as1(ptr addrspace(1) %in, ptr %out0, ptr %out1, ptr %out2, ptr %out3) { -; X64-LABEL: 'ptrtoaddr_as1' -; X64-NEXT: Classifying expressions for: @ptrtoaddr_as1 -; X64-NEXT: %p0 = ptrtoaddr ptr addrspace(1) %in to i64 -; X64-NEXT: --> %p0 U: full-set S: full-set -; X64-NEXT: %p1 = ptrtoaddr ptr addrspace(1) %in to i32 -; X64-NEXT: --> %p1 U: full-set S: full-set -; X64-NEXT: %p2 = ptrtoaddr ptr addrspace(1) %in to i16 -; X64-NEXT: --> %p2 U: full-set S: full-set -; X64-NEXT: %p3 = ptrtoaddr ptr addrspace(1) %in to i128 -; X64-NEXT: --> %p3 U: full-set S: full-set -; X64-NEXT: Determining loop execution counts for: @ptrtoaddr_as1 -; -; X32-LABEL: 'ptrtoaddr_as1' -; X32-NEXT: Classifying expressions for: @ptrtoaddr_as1 -; X32-NEXT: %p0 = ptrtoaddr ptr addrspace(1) %in to i64 -; X32-NEXT: --> %p0 U: full-set S: full-set -; X32-NEXT: %p1 = ptrtoaddr ptr addrspace(1) %in to i32 -; X32-NEXT: --> %p1 U: full-set S: full-set -; X32-NEXT: %p2 = ptrtoaddr ptr addrspace(1) %in to i16 -; X32-NEXT: --> %p2 U: full-set S: full-set -; X32-NEXT: %p3 = ptrtoaddr ptr addrspace(1) %in to i128 -; X32-NEXT: --> %p3 U: full-set S: full-set -; X32-NEXT: Determining loop execution counts for: @ptrtoaddr_as1 +; CHECK-LABEL: 'ptrtoaddr_as1' +; CHECK-NEXT: Classifying expressions for: @ptrtoaddr_as1 +; CHECK-NEXT: %p0 = ptrtoaddr ptr addrspace(1) %in to i64 +; CHECK-NEXT: --> (ptrtoaddr ptr addrspace(1) %in to i64) U: full-set S: full-set +; CHECK-NEXT: Determining loop execution counts for: @ptrtoaddr_as1 ; %p0 = ptrtoaddr ptr addrspace(1) %in to i64 - %p1 = ptrtoaddr ptr addrspace(1) %in to i32 - %p2 = ptrtoaddr ptr addrspace(1) %in to i16 - %p3 = ptrtoaddr ptr addrspace(1) %in to i128 store i64 %p0, ptr %out0 - store i32 %p1, ptr %out1 - store i16 %p2, ptr %out2 - store i128 %p3, ptr %out3 ret void } define void @ptrtoaddr_of_bitcast(ptr %in, ptr %out0) { -; X64-LABEL: 'ptrtoaddr_of_bitcast' -; X64-NEXT: Classifying expressions for: @ptrtoaddr_of_bitcast -; X64-NEXT: %in_casted = bitcast ptr %in to ptr -; X64-NEXT: --> %in U: full-set S: full-set -; X64-NEXT: %p0 = ptrtoaddr ptr %in_casted to i64 -; X64-NEXT: --> %p0 U: full-set S: full-set -; X64-NEXT: Determining loop execution counts for: @ptrtoaddr_of_bitcast -; -; X32-LABEL: 'ptrtoaddr_of_bitcast' -; X32-NEXT: Classifying expressions for: @ptrtoaddr_of_bitcast -; X32-NEXT: %in_casted = bitcast ptr %in to ptr -; X32-NEXT: --> %in U: full-set S: full-set -; X32-NEXT: %p0 = ptrtoaddr ptr %in_casted to i64 -; X32-NEXT: --> %p0 U: full-set S: full-set -; X32-NEXT: Determining loop execution counts for: @ptrtoaddr_of_bitcast +; CHECK-LABEL: 'ptrtoaddr_of_bitcast' +; CHECK-NEXT: Classifying expressions for: @ptrtoaddr_of_bitcast +; CHECK-NEXT: %in_casted = bitcast ptr %in to ptr +; CHECK-NEXT: --> %in U: full-set S: full-set +; CHECK-NEXT: %p0 = ptrtoaddr ptr %in_casted to i64 +; CHECK-NEXT: --> (ptrtoaddr ptr %in to i64) U: full-set S: full-set +; CHECK-NEXT: Determining loop execution counts for: @ptrtoaddr_of_bitcast ; %in_casted = bitcast ptr %in to ptr %p0 = ptrtoaddr ptr %in_casted to i64 @@ -100,11 +43,11 @@ define void @ptrtoaddr_of_bitcast(ptr %in, ptr %out0) { } define void @ptrtoaddr_of_nullptr(ptr %out0) { -; ALL-LABEL: 'ptrtoaddr_of_nullptr' -; ALL-NEXT: Classifying expressions for: @ptrtoaddr_of_nullptr -; ALL-NEXT: %p0 = ptrtoaddr ptr null to i64 -; ALL-NEXT: --> %p0 U: full-set S: full-set -; ALL-NEXT: Determining loop execution counts for: @ptrtoaddr_of_nullptr +; CHECK-LABEL: 'ptrtoaddr_of_nullptr' +; CHECK-NEXT: Classifying expressions for: @ptrtoaddr_of_nullptr +; CHECK-NEXT: %p0 = ptrtoaddr ptr null to i64 +; CHECK-NEXT: --> 0 U: [0,1) S: [0,1) +; CHECK-NEXT: Determining loop execution counts for: @ptrtoaddr_of_nullptr ; %p0 = ptrtoaddr ptr null to i64 store i64 %p0, ptr %out0 @@ -112,21 +55,13 @@ define void @ptrtoaddr_of_nullptr(ptr %out0) { } define void @ptrtoaddr_of_gep(ptr %in, ptr %out0) { -; X64-LABEL: 'ptrtoaddr_of_gep' -; X64-NEXT: Classifying expressions for: @ptrtoaddr_of_gep -; X64-NEXT: %in_adj = getelementptr inbounds i8, ptr %in, i64 42 -; X64-NEXT: --> (42 + %in) U: full-set S: full-set -; X64-NEXT: %p0 = ptrtoaddr ptr %in_adj to i64 -; X64-NEXT: --> %p0 U: full-set S: full-set -; X64-NEXT: Determining loop execution counts for: @ptrtoaddr_of_gep -; -; X32-LABEL: 'ptrtoaddr_of_gep' -; X32-NEXT: Classifying expressions for: @ptrtoaddr_of_gep -; X32-NEXT: %in_adj = getelementptr inbounds i8, ptr %in, i64 42 -; X32-NEXT: --> (42 + %in) U: full-set S: full-set -; X32-NEXT: %p0 = ptrtoaddr ptr %in_adj to i64 -; X32-NEXT: --> %p0 U: full-set S: full-set -; X32-NEXT: Determining loop execution counts for: @ptrtoaddr_of_gep +; CHECK-LABEL: 'ptrtoaddr_of_gep' +; CHECK-NEXT: Classifying expressions for: @ptrtoaddr_of_gep +; CHECK-NEXT: %in_adj = getelementptr inbounds i8, ptr %in, i64 42 +; CHECK-NEXT: --> (42 + %in) U: full-set S: full-set +; CHECK-NEXT: %p0 = ptrtoaddr ptr %in_adj to i64 +; CHECK-NEXT: --> (42 + (ptrtoaddr ptr %in to i64)) U: full-set S: full-set +; CHECK-NEXT: Determining loop execution counts for: @ptrtoaddr_of_gep ; %in_adj = getelementptr inbounds i8, ptr %in, i64 42 %p0 = ptrtoaddr ptr %in_adj to i64 diff --git a/llvm/test/Transforms/LoopVectorize/expand-ptrtoaddr.ll b/llvm/test/Transforms/LoopVectorize/expand-ptrtoaddr.ll new file mode 100644 index 0000000000000..e925e8422c3fd --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/expand-ptrtoaddr.ll @@ -0,0 +1,47 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph:" --version 6 +; RUN: opt -p loop-vectorize -force-vector-width=4 -S %s | FileCheck %s + +define void @test_ptrtoaddr_tripcount(ptr %start, ptr %end) { +; CHECK-LABEL: define void @test_ptrtoaddr_tripcount( +; CHECK-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[START_ADDR:%.*]] = ptrtoaddr ptr [[START]] to i64 +; CHECK-NEXT: [[END_ADDR:%.*]] = ptrtoaddr ptr [[END]] to i64 +; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[END_ADDR]], 1 +; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START_ADDR]] +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4 +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[START_ADDR]], [[N_VEC]] +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[START_ADDR]], [[INDEX]] +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[START]], i64 [[OFFSET_IDX]] +; CHECK-NEXT: store <4 x i8> zeroinitializer, ptr [[TMP3]], align 1 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]] +; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] +; CHECK: [[SCALAR_PH]]: +; +entry: + %start.addr = ptrtoaddr ptr %start to i64 + %end.addr = ptrtoaddr ptr %end to i64 + br label %loop + +loop: + %iv = phi i64 [ %start.addr, %entry ], [ %iv.next, %loop ] + %gep = getelementptr inbounds i8, ptr %start, i64 %iv + store i8 0, ptr %gep + %iv.next = add i64 %iv, 1 + %cmp = icmp ne i64 %iv, %end.addr + br i1 %cmp, label %loop, label %exit + +exit: + ret void +} diff --git a/polly/include/polly/Support/SCEVAffinator.h b/polly/include/polly/Support/SCEVAffinator.h index faacfd8ba0e69..5fa78cc2431e7 100644 --- a/polly/include/polly/Support/SCEVAffinator.h +++ b/polly/include/polly/Support/SCEVAffinator.h @@ -100,6 +100,7 @@ class SCEVAffinator final : public llvm::SCEVVisitor { PWACtx visit(const llvm::SCEV *E); PWACtx visitConstant(const llvm::SCEVConstant *E); PWACtx visitVScale(const llvm::SCEVVScale *E); + PWACtx visitPtrToAddrExpr(const llvm::SCEVPtrToAddrExpr *E); PWACtx visitPtrToIntExpr(const llvm::SCEVPtrToIntExpr *E); PWACtx visitTruncateExpr(const llvm::SCEVTruncateExpr *E); PWACtx visitZeroExtendExpr(const llvm::SCEVZeroExtendExpr *E); diff --git a/polly/lib/Support/SCEVAffinator.cpp b/polly/lib/Support/SCEVAffinator.cpp index 87e0fc056ca4b..c9a728d3d04ec 100644 --- a/polly/lib/Support/SCEVAffinator.cpp +++ b/polly/lib/Support/SCEVAffinator.cpp @@ -271,6 +271,10 @@ PWACtx SCEVAffinator::visitVScale(const SCEVVScale *VScale) { llvm_unreachable("SCEVVScale not yet supported"); } +PWACtx SCEVAffinator::visitPtrToAddrExpr(const SCEVPtrToAddrExpr *Expr) { + return visit(Expr->getOperand(0)); +} + PWACtx SCEVAffinator::visitPtrToIntExpr(const SCEVPtrToIntExpr *Expr) { return visit(Expr->getOperand(0)); } diff --git a/polly/lib/Support/SCEVValidator.cpp b/polly/lib/Support/SCEVValidator.cpp index ad3d0c22295b5..b091ed2cd550a 100644 --- a/polly/lib/Support/SCEVValidator.cpp +++ b/polly/lib/Support/SCEVValidator.cpp @@ -156,6 +156,10 @@ class SCEVValidator : public SCEVVisitor { return ValidatorResult(SCEVType::PARAM, Expr); } + ValidatorResult visitPtrToAddrExpr(const SCEVPtrToAddrExpr *Expr) { + return visit(Expr->getOperand()); + } + ValidatorResult visitPtrToIntExpr(const SCEVPtrToIntExpr *Expr) { return visit(Expr->getOperand()); } diff --git a/polly/lib/Support/ScopHelper.cpp b/polly/lib/Support/ScopHelper.cpp index a2328d1bbb3cf..6f9956d29c063 100644 --- a/polly/lib/Support/ScopHelper.cpp +++ b/polly/lib/Support/ScopHelper.cpp @@ -379,6 +379,9 @@ struct ScopExpander final : SCEVVisitor { ///{ const SCEV *visitConstant(const SCEVConstant *E) { return E; } const SCEV *visitVScale(const SCEVVScale *E) { return E; } + const SCEV *visitPtrToAddrExpr(const SCEVPtrToAddrExpr *E) { + return GenSE.getPtrToAddrExpr(visit(E->getOperand())); + } const SCEV *visitPtrToIntExpr(const SCEVPtrToIntExpr *E) { return GenSE.getPtrToIntExpr(visit(E->getOperand()), E->getType()); }