diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h index 80ef32aff62ae..3c748b701c9ff 100644 --- a/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -1502,6 +1502,11 @@ enum NodeType { // Outputs: [rv], output chain, glue PATCHPOINT, + // PTRADD represents pointer arithmetic semantics, for targets that opt in + // using shouldPreservePtrArith(). + // ptr = PTRADD ptr, offset + PTRADD, + // Vector Predication #define BEGIN_REGISTER_VP_SDNODE(VPSDID, ...) VPSDID, #include "llvm/IR/VPIntrinsics.def" diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index cfefceea8f0fe..b7a527b81019d 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -213,6 +213,7 @@ class SDValue { inline bool isTargetOpcode() const; inline bool isMachineOpcode() const; inline bool isUndef() const; + inline bool isAnyAdd() const; inline unsigned getMachineOpcode() const; inline const DebugLoc &getDebugLoc() const; inline void dump() const; @@ -697,6 +698,11 @@ END_TWO_BYTE_PACK() return NodeType == ISD::UNDEF || NodeType == ISD::POISON; } + /// Returns true if the node type is ADD or PTRADD. + bool isAnyAdd() const { + return NodeType == ISD::ADD || NodeType == ISD::PTRADD; + } + /// Test if this node is a memory intrinsic (with valid pointer information). bool isMemIntrinsic() const { return SDNodeBits.IsMemIntrinsic; } @@ -1270,6 +1276,8 @@ inline bool SDValue::isUndef() const { return Node->isUndef(); } +inline bool SDValue::isAnyAdd() const { return Node->isAnyAdd(); } + inline bool SDValue::use_empty() const { return !Node->hasAnyUseOfValue(ResNo); } diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 03099e9ad44dc..f9d3d82ab4a84 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -3483,6 +3483,15 @@ class TargetLoweringBase { /// doing arithmetic on boolean types virtual bool shouldExpandCmpUsingSelects(EVT VT) const { return false; } + /// True if target has some particular form of dealing with pointer arithmetic + /// semantics for pointers with the given value type. False if pointer + /// arithmetic should not be preserved for passes such as instruction + /// selection, and can fallback to regular arithmetic. + /// This should be removed when PTRADD nodes are widely supported by backends. + virtual bool shouldPreservePtrArith(const Function &F, EVT PtrVT) const { + return false; + } + /// Does this target support complex deinterleaving virtual bool isComplexDeinterleavingSupported() const { return false; } diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td index 41fed692c7025..349fd003ca06e 100644 --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -407,7 +407,7 @@ def tblockaddress: SDNode<"ISD::TargetBlockAddress", SDTPtrLeaf, [], def add : SDNode<"ISD::ADD" , SDTIntBinOp , [SDNPCommutative, SDNPAssociative]>; -def ptradd : SDNode<"ISD::ADD" , SDTPtrAddOp, []>; +def ptradd : SDNode<"ISD::PTRADD" , SDTPtrAddOp, []>; def sub : SDNode<"ISD::SUB" , SDTIntBinOp>; def mul : SDNode<"ISD::MUL" , SDTIntBinOp, [SDNPCommutative, SDNPAssociative]>; diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index d6e288a59b2ee..350895208a4fb 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -412,7 +412,8 @@ namespace { SDValue visitMERGE_VALUES(SDNode *N); SDValue visitADD(SDNode *N); SDValue visitADDLike(SDNode *N); - SDValue visitADDLikeCommutative(SDValue N0, SDValue N1, SDNode *LocReference); + SDValue visitADDLikeCommutative(SDValue N0, SDValue N1, + SDNode *LocReference); SDValue visitSUB(SDNode *N); SDValue visitADDSAT(SDNode *N); SDValue visitSUBSAT(SDNode *N); @@ -1095,7 +1096,7 @@ bool DAGCombiner::reassociationCanBreakAddressingModePattern(unsigned Opc, // (load/store (add, (add, x, y), offset2)) -> // (load/store (add, (add, x, offset2), y)). - if (N0.getOpcode() != ISD::ADD) + if (!N0.isAnyAdd()) return false; // Check for vscale addressing modes. @@ -2388,7 +2389,7 @@ static bool canFoldInAddressingMode(SDNode *N, SDNode *Use, SelectionDAG &DAG, } TargetLowering::AddrMode AM; - if (N->getOpcode() == ISD::ADD) { + if (N->isAnyAdd()) { AM.HasBaseReg = true; ConstantSDNode *Offset = dyn_cast(N->getOperand(1)); if (Offset) diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index bbf1b0fd590ef..ffedec2975a7b 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -5641,7 +5641,7 @@ bool SelectionDAG::isADDLike(SDValue Op, bool NoWrap) const { bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { return Op.getNumOperands() == 2 && isa(Op.getOperand(1)) && - (Op.getOpcode() == ISD::ADD || isADDLike(Op)); + (Op.isAnyAdd() || isADDLike(Op)); } bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, @@ -7341,10 +7341,18 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, case ISD::OR: case ISD::XOR: case ISD::ADD: + case ISD::PTRADD: case ISD::SUB: assert(VT.isInteger() && "This operator does not apply to FP types!"); assert(N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && "Binary operator types must match!"); + // The equal operand types requirement is unnecessarily strong for PTRADD. + // However, the SelectionDAGBuilder does not generate PTRADDs with different + // operand types, and we'd need to re-implement GEP's non-standard wrapping + // logic everywhere where PTRADDs may be folded or combined to properly + // support them. If/when we introduce pointer types to the SDAG, we will + // need to relax this constraint. + // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so // it's worth handling here. if (N2CV && N2CV->isZero()) @@ -7696,6 +7704,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, std::swap(N1, N2); } else { switch (Opcode) { + case ISD::PTRADD: case ISD::SUB: // fold op(undef, arg2) -> undef, fold op(poison, arg2) ->poison. return N1.getOpcode() == ISD::POISON ? getPOISON(VT) : getUNDEF(VT); @@ -7723,6 +7732,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, return getConstant(0, DL, VT); [[fallthrough]]; case ISD::ADD: + case ISD::PTRADD: case ISD::SUB: case ISD::UDIV: case ISD::SDIV: @@ -8144,6 +8154,9 @@ SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset, const SDNodeFlags Flags) { assert(Offset.getValueType().isInteger()); EVT BasePtrVT = Ptr.getValueType(); + if (TLI->shouldPreservePtrArith(this->getMachineFunction().getFunction(), + BasePtrVT)) + return getNode(ISD::PTRADD, DL, BasePtrVT, Ptr, Offset, Flags); return getNode(ISD::ADD, DL, BasePtrVT, Ptr, Offset, Flags); } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 8e74a076cc013..6459c0b9be30c 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -4284,8 +4284,8 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) { (int64_t(Offset) >= 0 && NW.hasNoUnsignedSignedWrap())) Flags |= SDNodeFlags::NoUnsignedWrap; - N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, - DAG.getConstant(Offset, dl, N.getValueType()), Flags); + N = DAG.getMemBasePlusOffset( + N, DAG.getConstant(Offset, dl, N.getValueType()), dl, Flags); } } else { // IdxSize is the width of the arithmetic according to IR semantics. @@ -4329,7 +4329,7 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) { OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType()); - N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags); + N = DAG.getMemBasePlusOffset(N, OffsVal, dl, Flags); continue; } @@ -4389,7 +4389,7 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) { SDNodeFlags AddFlags; AddFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap()); - N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, IdxN, AddFlags); + N = DAG.getMemBasePlusOffset(N, IdxN, dl, AddFlags); } } @@ -9138,8 +9138,7 @@ bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) { Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType()); // Adjust return pointer to point just past the last dst byte. - SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(), - Dst, Size); + SDValue DstPlusSize = DAG.getMemBasePlusOffset(Dst, Size, sdl); setValue(&I, DstPlusSize); return true; } @@ -11230,10 +11229,9 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { MachineFunction &MF = CLI.DAG.getMachineFunction(); Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx); for (unsigned i = 0; i < NumValues; ++i) { - SDValue Add = - CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot, - CLI.DAG.getConstant(Offsets[i], CLI.DL, PtrVT), - SDNodeFlags::NoUnsignedWrap); + SDValue Add = CLI.DAG.getMemBasePlusOffset( + DemoteStackSlot, CLI.DAG.getConstant(Offsets[i], CLI.DL, PtrVT), + CLI.DL, SDNodeFlags::NoUnsignedWrap); SDValue L = CLI.DAG.getLoad( RetTys[i], CLI.DL, CLI.Chain, Add, MachinePointerInfo::getFixedStack(CLI.DAG.getMachineFunction(), diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp index 8faf97271d99e..45ef475dffe6e 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -269,6 +269,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { // Binary operators case ISD::ADD: return "add"; + case ISD::PTRADD: return "ptradd"; case ISD::SUB: return "sub"; case ISD::MUL: return "mul"; case ISD::MULHU: return "mulhu";