Skip to content

[FPEnv][SDAG] Implement FNEARBYINT with optional chain #163081

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: users/spavloff/op-chain
Choose a base branch
from
Open

[FPEnv][SDAG] Implement FNEARBYINT with optional chain #163081

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion llvm/include/llvm/CodeGen/ISDOpcodes.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -448,7 +448,6 @@ enum NodeType {
STRICT_FLOG10,
STRICT_FLOG2,
STRICT_FRINT,
STRICT_FNEARBYINT,
STRICT_FMAXNUM,
STRICT_FMINNUM,
STRICT_FCEIL,
Expand Down
2 changes: 2 additions & 0 deletions llvm/include/llvm/CodeGen/TargetLowering.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1341,6 +1341,8 @@ class LLVM_ABI TargetLoweringBase {
unsigned EqOpc;
switch (Op) {
default: llvm_unreachable("Unexpected FP pseudo-opcode");
#define FP_OPERATION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
case ISD::DAGN: EqOpc = ISD::DAGN; break;
#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
case ISD::STRICT_##DAGN: EqOpc = ISD::DAGN; break;
#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
Expand Down
9 changes: 8 additions & 1 deletion llvm/include/llvm/IR/ConstrainedOps.def
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,12 @@
#define CMP_INSTRUCTION(N,A,R,I,D) DAG_INSTRUCTION(N,A,R,I,D)
#endif

// FP_OPERATION is same as DAG_FUNCTION, but in DAG it is represented by the
// same node, as non-constrained function.
#ifndef FP_OPERATION
#define FP_OPERATION(N,A,R,I,D) DAG_FUNCTION(N,A,R,I,D)
#endif

// Arguments of the entries are:
// - instruction or intrinsic function name.
// - Number of original instruction/intrinsic arguments.
Expand Down Expand Up @@ -91,7 +97,7 @@ DAG_FUNCTION(maxnum, 2, 0, experimental_constrained_maxnum, FMAXNUM
DAG_FUNCTION(minnum, 2, 0, experimental_constrained_minnum, FMINNUM)
DAG_FUNCTION(maximum, 2, 0, experimental_constrained_maximum, FMAXIMUM)
DAG_FUNCTION(minimum, 2, 0, experimental_constrained_minimum, FMINIMUM)
DAG_FUNCTION(nearbyint, 1, 1, experimental_constrained_nearbyint, FNEARBYINT)
FP_OPERATION(nearbyint, 1, 1, experimental_constrained_nearbyint, FNEARBYINT)
DAG_FUNCTION(pow, 2, 1, experimental_constrained_pow, FPOW)
DAG_FUNCTION(powi, 2, 1, experimental_constrained_powi, FPOWI)
DAG_FUNCTION(ldexp, 2, 1, experimental_constrained_ldexp, FLDEXP)
Expand All @@ -114,3 +120,4 @@ FUNCTION(fmuladd, 3, 1, experimental_constrained_fmuladd)
#undef CMP_INSTRUCTION
#undef DAG_INSTRUCTION
#undef DAG_FUNCTION
#undef FP_OPERATION
7 changes: 1 addition & 6 deletions llvm/include/llvm/Target/TargetSelectionDAG.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -575,7 +575,7 @@ def frint : SDNode<"ISD::FRINT" , SDTFPUnaryOp>;
def ftrunc : SDNode<"ISD::FTRUNC" , SDTFPUnaryOp>;
def fceil : SDNode<"ISD::FCEIL" , SDTFPUnaryOp>;
def ffloor : SDNode<"ISD::FFLOOR" , SDTFPUnaryOp>;
def fnearbyint : SDNode<"ISD::FNEARBYINT" , SDTFPUnaryOp>;
def fnearbyint : SDNode<"ISD::FNEARBYINT" , SDTFPUnaryOp, [SDNPMayHaveChain]>;
def fround : SDNode<"ISD::FROUND" , SDTFPUnaryOp>;
def froundeven : SDNode<"ISD::FROUNDEVEN" , SDTFPUnaryOp>;

Expand Down Expand Up @@ -653,8 +653,6 @@ def strict_lrint : SDNode<"ISD::STRICT_LRINT",
SDTFPToIntOp, [SDNPHasChain]>;
def strict_llrint : SDNode<"ISD::STRICT_LLRINT",
SDTFPToIntOp, [SDNPHasChain]>;
def strict_fnearbyint : SDNode<"ISD::STRICT_FNEARBYINT",
SDTFPUnaryOp, [SDNPHasChain]>;
def strict_fceil : SDNode<"ISD::STRICT_FCEIL",
SDTFPUnaryOp, [SDNPHasChain]>;
def strict_ffloor : SDNode<"ISD::STRICT_FFLOOR",
Expand Down Expand Up @@ -1704,9 +1702,6 @@ def any_lrint : PatFrags<(ops node:$src),
def any_llrint : PatFrags<(ops node:$src),
[(strict_llrint node:$src),
(llrint node:$src)]>;
def any_fnearbyint : PatFrags<(ops node:$src),
[(strict_fnearbyint node:$src),
(fnearbyint node:$src)]>;
def any_fceil : PatFrags<(ops node:$src),
[(strict_fceil node:$src),
(fceil node:$src)]>;
Expand Down
24 changes: 20 additions & 4 deletions llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2196,7 +2196,7 @@ void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
if (LC == RTLIB::UNKNOWN_LIBCALL)
llvm_unreachable("Can't create an unknown libcall!");

if (Node->isStrictFPOpcode()) {
if (Node->isStrictFPOpcode() || (Node->hasChain() && Node->isFPOperation())) {
EVT RetVT = Node->getValueType(0);
SmallVector<SDValue, 4> Ops(drop_begin(Node->ops()));
TargetLowering::MakeLibCallOptions CallOptions;
Expand Down Expand Up @@ -4791,7 +4791,6 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
RTLIB::RINT_PPCF128, Results);
break;
case ISD::FNEARBYINT:
case ISD::STRICT_FNEARBYINT:
ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
RTLIB::NEARBYINT_F64,
RTLIB::NEARBYINT_F80,
Expand Down Expand Up @@ -5760,7 +5759,6 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
case ISD::FFLOOR:
case ISD::FCEIL:
case ISD::FRINT:
case ISD::FNEARBYINT:
case ISD::FROUND:
case ISD::FROUNDEVEN:
case ISD::FTRUNC:
Expand Down Expand Up @@ -5792,7 +5790,6 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
case ISD::STRICT_FFLOOR:
case ISD::STRICT_FCEIL:
case ISD::STRICT_FRINT:
case ISD::STRICT_FNEARBYINT:
case ISD::STRICT_FROUND:
case ISD::STRICT_FROUNDEVEN:
case ISD::STRICT_FTRUNC:
Expand Down Expand Up @@ -5821,6 +5818,25 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
Results.push_back(Tmp3);
Results.push_back(Tmp3.getValue(1));
break;
case ISD::FNEARBYINT:
if (Node->hasChain()) {
Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
{Node->getOperand(0), Node->getOperand(1)});
Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
{Tmp1.getValue(1), Tmp1});
Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
{Tmp2.getValue(1), Tmp2,
DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)});
Results.push_back(Tmp3);
Results.push_back(Tmp3.getValue(1));
} else {
Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
Results.push_back(
DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2,
DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
}
break;
case ISD::BUILD_VECTOR: {
MVT EltVT = OVT.getVectorElementType();
MVT NewEltVT = NVT.getVectorElementType();
Expand Down
114 changes: 108 additions & 6 deletions llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,6 @@ void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
case ISD::STRICT_FMUL:
case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
case ISD::STRICT_FNEARBYINT:
case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
case ISD::STRICT_FP_EXTEND:
Expand Down Expand Up @@ -227,6 +226,32 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
return Tmp.first;
}

SDValue DAGTypeLegalizer::SoftenFloatRes_FPOperation(SDNode *N,
RTLIB::Libcall LC) {
bool HasChain = N->hasChain();
assert(N->getNumValues() == 1 + HasChain &&
"multiple result is not supported yet");
SDValue Chain = HasChain ? N->getOperand(0) : SDValue();
SmallVector<SDValue, 4> Ops;
SmallVector<EVT, 4> OpsVT;

for (unsigned i = HasChain, e = N->getNumOperands(); i != e; ++i) {
SDValue Op = N->getOperand(i);
OpsVT.push_back(Op.getValueType());
Op = GetSoftenedFloat(Op);
Ops.push_back(Op);
}

EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
TargetLowering::MakeLibCallOptions CallOptions;
CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0));
std::pair<SDValue, SDValue> Tmp =
TLI.makeLibCall(DAG, LC, NVT, Ops, CallOptions, SDLoc(N), Chain);
if (HasChain)
ReplaceValueWith(SDValue(N, 1), Tmp.second);
return Tmp.first;
}

SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
return BitConvertToInteger(N->getOperand(0));
}
Expand Down Expand Up @@ -582,7 +607,7 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
}

SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
return SoftenFloatRes_FPOperation(N, GetFPLibCall(N->getValueType(0),
RTLIB::NEARBYINT_F32,
RTLIB::NEARBYINT_F64,
RTLIB::NEARBYINT_F80,
Expand Down Expand Up @@ -1596,7 +1621,6 @@ void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
case ISD::STRICT_FMUL:
case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
case ISD::STRICT_FNEARBYINT:
case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
case ISD::STRICT_FP_EXTEND:
Expand Down Expand Up @@ -1688,6 +1712,21 @@ void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
GetPairElements(Tmp.first, Lo, Hi);
}

void DAGTypeLegalizer::ExpandFloatRes_FPOperation(SDNode *N, RTLIB::Libcall LC,
SDValue &Lo, SDValue &Hi) {
bool HasChain = N->hasChain();
SDValue Chain = HasChain ? N->getOperand(0) : SDValue();
assert(N->getNumValues() == 1 + HasChain &&
"multiple result is not supported yet");
SmallVector<SDValue, 4> Ops(HasChain ? llvm::drop_begin(N->ops()) : N->ops());
TargetLowering::MakeLibCallOptions CallOptions;
std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
DAG, LC, N->getValueType(0), Ops, CallOptions, SDLoc(N), Chain);
if (HasChain)
ReplaceValueWith(SDValue(N, 1), Tmp.second);
GetPairElements(Tmp.first, Lo, Hi);
}

void DAGTypeLegalizer::ExpandFloatRes_FMODF(SDNode *N) {
ExpandFloatRes_UnaryWithTwoFPResults(N, RTLIB::getMODF(N->getValueType(0)),
/*CallRetResNo=*/0);
Expand Down Expand Up @@ -1951,7 +1990,7 @@ void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,

void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
SDValue &Lo, SDValue &Hi) {
ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
ExpandFloatRes_FPOperation(N, GetFPLibCall(N->getValueType(0),
RTLIB::NEARBYINT_F32,
RTLIB::NEARBYINT_F64,
RTLIB::NEARBYINT_F80,
Expand Down Expand Up @@ -2827,6 +2866,11 @@ void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;

// Floating-point operations with optional chain.
case ISD::FNEARBYINT:
R = PromoteFloatRes_FPOperation(N);
break;

// Unary FP Operations
case ISD::FABS:
case ISD::FACOS:
Expand All @@ -2843,7 +2887,6 @@ void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
case ISD::FLOG:
case ISD::FLOG2:
case ISD::FLOG10:
case ISD::FNEARBYINT:
case ISD::FNEG:
case ISD::FRINT:
case ISD::FROUND:
Expand Down Expand Up @@ -3071,6 +3114,29 @@ SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
}

SDValue DAGTypeLegalizer::PromoteFloatRes_FPOperation(SDNode *N) {
bool HasChain = N->hasChain();
SDValue Chain = HasChain ? N->getOperand(0) : SDValue();
assert(N->getNumValues() == 1 + HasChain &&
"multiple result is not supported yet");
SmallVector<SDValue, 4> Ops;

if (HasChain)
Ops.push_back(Chain);
for (unsigned i = HasChain, e = N->getNumOperands(); i != e; ++i) {
SDValue Op = N->getOperand(i);
// FIXME Use strict conversions for strict operations.
Op = GetPromotedFloat(Op);
Ops.push_back(Op);
}

EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Ops);
if (HasChain)
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
return Res;
}

SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
EVT VT = N->getValueType(0);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
Expand Down Expand Up @@ -3312,6 +3378,11 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
case ISD::STRICT_FP_ROUND:
case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;

// Floating-point operations with optional chain.
case ISD::FNEARBYINT:
R = SoftPromoteHalfRes_FPOperation(N);
break;

// Unary FP Operations
case ISD::FACOS:
case ISD::FASIN:
Expand All @@ -3327,7 +3398,6 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
case ISD::FLOG:
case ISD::FLOG2:
case ISD::FLOG10:
case ISD::FNEARBYINT:
case ISD::FREEZE:
case ISD::FRINT:
case ISD::FROUND:
Expand Down Expand Up @@ -3714,6 +3784,38 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
}

SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FPOperation(SDNode *N) {
SDLoc dl(N);
bool HasChain = N->hasChain();
assert(N->getNumValues() == 1 + HasChain &&
"multiple result is not supported yet");
SDValue Chain = HasChain ? N->getOperand(0) : SDValue();
EVT OVT = N->getValueType(0);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);

SmallVector<SDValue, 4> Ops;
if (HasChain)
Ops.push_back(Chain);
for (unsigned i = HasChain, e = N->getNumOperands(); i != e; ++i) {
SDValue Op = GetSoftPromotedHalf(N->getOperand(i));
// FIXME Use strict conversions for strict operations.
Op = DAG.getNode(PromotionOpcode, dl, NVT, Op);
Ops.push_back(Op);
}

SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Ops);
if (HasChain)
Chain = Res.getValue(1);

// Convert back to FP16 as an integer.
Res = DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);

if (HasChain)
ReplaceValueWith(SDValue(N, 1), Chain);
return Res;
}

SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
// Expand and soften recursively.
ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
Expand Down
7 changes: 7 additions & 0 deletions llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -573,6 +573,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
bool SoftenFloatRes_UnaryWithTwoFPResults(
SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo = {});
SDValue SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC);
SDValue SoftenFloatRes_FPOperation(SDNode *N, RTLIB::Libcall LC);
SDValue SoftenFloatRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
SDValue SoftenFloatRes_ARITH_FENCE(SDNode *N);
SDValue SoftenFloatRes_BITCAST(SDNode *N);
Expand Down Expand Up @@ -681,6 +682,8 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue &Lo, SDValue &Hi);
void ExpandFloatRes_UnaryWithTwoFPResults(
SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo = {});
void ExpandFloatRes_FPOperation(SDNode *N, RTLIB::Libcall LC, SDValue &Lo,
SDValue &Hi);

// clang-format off
void ExpandFloatRes_AssertNoFPClass(SDNode *N, SDValue &Lo, SDValue &Hi);
Expand Down Expand Up @@ -788,6 +791,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteFloatRes_XINT_TO_FP(SDNode *N);
SDValue PromoteFloatRes_VECREDUCE(SDNode *N);
SDValue PromoteFloatRes_VECREDUCE_SEQ(SDNode *N);
SDValue PromoteFloatRes_FPOperation(SDNode *N);

bool PromoteFloatOperand(SDNode *N, unsigned OpNo);
SDValue PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo);
Expand Down Expand Up @@ -839,6 +843,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue SoftPromoteHalfRes_UNDEF(SDNode *N);
SDValue SoftPromoteHalfRes_VECREDUCE(SDNode *N);
SDValue SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N);
SDValue SoftPromoteHalfRes_FPOperation(SDNode *N);

bool SoftPromoteHalfOperand(SDNode *N, unsigned OpNo);
SDValue SoftPromoteHalfOp_BITCAST(SDNode *N);
Expand Down Expand Up @@ -881,6 +886,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue ScalarizeVecRes_OverflowOp(SDNode *N, unsigned ResNo);
SDValue ScalarizeVecRes_InregOp(SDNode *N);
SDValue ScalarizeVecRes_VecInregOp(SDNode *N);
SDValue ScalarizeVecRes_FPOperation(SDNode *N);

SDValue ScalarizeVecRes_ADDRSPACECAST(SDNode *N);
SDValue ScalarizeVecRes_BITCAST(SDNode *N);
Expand Down Expand Up @@ -965,6 +971,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
void SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
SDValue &Lo, SDValue &Hi);
void SplitVecRes_FPOperation(SDNode *N, SDValue &Lo, SDValue &Hi);

void SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi);

Expand Down
Loading
Loading