Skip to content

Commit 821d282

Browse files
arsenmarsenm
authored
RuntimeLibcalls: Remove incorrect sincospi from most targets (llvm#166982)
sincospi/sincospif/sincospil does not appear to exist on common targets. Darwin targets have __sincospi and __sincospif, so define and use those implementations. I have no idea what version added those calls, so I'm just guessing it's the same conditions as __sincos_stret. Most of this patch is working to preserve codegen when a vector library is explicitly enabled. This only covers sleef and armpl, as those are the only cases tested. The multiple result libcalls have an aberrant process where the legalizer looks for the scalar type's libcall in RuntimeLibcalls, and then cross references TargetLibraryInfo to find a matching vector call. This was unworkable in the sincospi case, since the common case is there is no scalar call available. To preserve codegen if the call is available, first try to match a libcall with the vector type before falling back on the old scalar search. Eventually all of this logic should be contained in RuntimeLibcalls, without the link to TargetLibraryInfo. In principle we should perform the same legalization logic as for an ordinary operation, trying to find a matching subvector type with a libcall.
1 parent 46a8ddb commit 821d282

File tree

15 files changed

+801
-274
lines changed

15 files changed

+801
-274
lines changed

‎llvm/include/llvm/CodeGen/BasicTTIImpl.h‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -316,12 +316,22 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
316316
EVT ScalarVT = VT.getScalarType();
317317
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
318318

319+
/// Migration flag. IsVectorCall cases directly know about the vector
320+
/// libcall in RuntimeLibcallsInfo and shouldn't try to use
321+
/// LibInfo->getVectorMappingInfo.
322+
bool IsVectorCall = false;
323+
319324
switch (ICA.getID()) {
320325
case Intrinsic::modf:
321326
LC = RTLIB::getMODF(ScalarVT);
322327
break;
323328
case Intrinsic::sincospi:
324-
LC = RTLIB::getSINCOSPI(ScalarVT);
329+
LC = RTLIB::getSINCOSPI(VT);
330+
if (LC == RTLIB::UNKNOWN_LIBCALL)
331+
LC = RTLIB::getSINCOSPI(ScalarVT);
332+
else if (VT.isVector())
333+
IsVectorCall = true;
334+
325335
break;
326336
case Intrinsic::sincos:
327337
LC = RTLIB::getSINCOS(ScalarVT);
@@ -345,17 +355,23 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
345355
LLVMContext &Ctx = RetTy->getContext();
346356
ElementCount VF = getVectorizedTypeVF(RetTy);
347357
VecDesc const *VD = nullptr;
348-
for (bool Masked : {false, true}) {
349-
if ((VD = LibInfo->getVectorMappingInfo(LCName, VF, Masked)))
350-
break;
358+
359+
if (!IsVectorCall) {
360+
for (bool Masked : {false, true}) {
361+
if ((VD = LibInfo->getVectorMappingInfo(LCName, VF, Masked)))
362+
break;
363+
}
364+
if (!VD)
365+
return std::nullopt;
351366
}
352-
if (!VD)
353-
return std::nullopt;
354367

355368
// Cost the call + mask.
356369
auto Cost =
357370
thisT()->getCallInstrCost(nullptr, RetTy, ICA.getArgTypes(), CostKind);
358-
if (VD->isMasked()) {
371+
372+
if ((VD && VD->isMasked()) ||
373+
(IsVectorCall &&
374+
RTLIB::RuntimeLibcallsInfo::hasVectorMaskArgument(LibcallImpl))) {
359375
auto VecTy = VectorType::get(IntegerType::getInt1Ty(Ctx), VF);
360376
Cost += thisT()->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy,
361377
VecTy, {}, CostKind, 0, nullptr, {});

‎llvm/include/llvm/CodeGen/SelectionDAG.h‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1725,9 +1725,17 @@ class SelectionDAG {
17251725
/// value.
17261726
LLVM_ABI bool
17271727
expandMultipleResultFPLibCall(RTLIB::Libcall LC, SDNode *Node,
1728-
SmallVectorImpl<SDValue> &Results,
1728+
SmallVectorImpl<SDValue> &Results, EVT CallType,
17291729
std::optional<unsigned> CallRetResNo = {});
17301730

1731+
// FIXME: Ths should be removed, and form using RTLIB::Libcall should be
1732+
// preferred. Callers should resolve the exact type libcall to use.
1733+
LLVM_ABI bool
1734+
expandMultipleResultFPLibCall(StringRef LibcallName, CallingConv::ID CC,
1735+
SDNode *Node, SmallVectorImpl<SDValue> &Results,
1736+
std::optional<unsigned> CallRetResNo = {},
1737+
bool IsVectorMasked = false);
1738+
17311739
/// Expand the specified \c ISD::VAARG node as the Legalize pass would.
17321740
LLVM_ABI SDValue expandVAArg(SDNode *Node);
17331741

‎llvm/include/llvm/IR/RuntimeLibcalls.td‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -975,10 +975,6 @@ def frexpf : RuntimeLibcallImpl<FREXP_F32>;
975975
def frexp : RuntimeLibcallImpl<FREXP_F64>;
976976
defm frexpl : LibmLongDoubleLibCall;
977977

978-
def sincospif : RuntimeLibcallImpl<SINCOSPI_F32>;
979-
def sincospi : RuntimeLibcallImpl<SINCOSPI_F64>;
980-
defm sincospil : LibmLongDoubleLibCall;
981-
982978
def modff : RuntimeLibcallImpl<MODF_F32>;
983979
def modf : RuntimeLibcallImpl<MODF_F64>;
984980
defm modfl : LibmLongDoubleLibCall;
@@ -1055,6 +1051,15 @@ def sincosf : RuntimeLibcallImpl<SINCOS_F32>;
10551051
def sincos : RuntimeLibcallImpl<SINCOS_F64>;
10561052
defm sincosl : LibmLongDoubleLibCall;
10571053

1054+
// Exists in sun math library
1055+
def sincospif : RuntimeLibcallImpl<SINCOSPI_F32>;
1056+
def sincospi : RuntimeLibcallImpl<SINCOSPI_F64>;
1057+
defm sincospil : LibmLongDoubleLibCall;
1058+
1059+
// Exists on macOS
1060+
def __sincospif : RuntimeLibcallImpl<SINCOSPI_F32>;
1061+
def __sincospi : RuntimeLibcallImpl<SINCOSPI_F64>;
1062+
10581063
def bzero : RuntimeLibcallImpl<BZERO>;
10591064
def __bzero : RuntimeLibcallImpl<BZERO>;
10601065

@@ -1232,7 +1237,9 @@ defvar DefaultLibcallImpls32 = (add DefaultRuntimeLibcallImpls);
12321237
defvar DefaultLibcallImpls64 = (add DefaultRuntimeLibcallImpls,
12331238
Int128RTLibcalls);
12341239

1235-
defvar DarwinSinCosStret = LibcallImpls<(add __sincosf_stret, __sincos_stret),
1240+
// TODO: Guessing sincospi added at same time as sincos_stret
1241+
defvar DarwinSinCosStret = LibcallImpls<(add __sincosf_stret, __sincos_stret,
1242+
__sincospif, __sincospi),
12361243
darwinHasSinCosStret>;
12371244
defvar DarwinExp10 = LibcallImpls<(add __exp10f, __exp10), darwinHasExp10>;
12381245

‎llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4842,9 +4842,15 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
48424842
RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
48434843
? RTLIB::getSINCOS(VT)
48444844
: RTLIB::getSINCOSPI(VT);
4845-
bool Expanded = DAG.expandMultipleResultFPLibCall(LC, Node, Results);
4846-
if (!Expanded)
4847-
llvm_unreachable("Expected scalar FSINCOS[PI] to expand to libcall!");
4845+
bool Expanded = DAG.expandMultipleResultFPLibCall(LC, Node, Results, VT);
4846+
if (!Expanded) {
4847+
DAG.getContext()->emitError(Twine("no libcall available for ") +
4848+
Node->getOperationName(&DAG));
4849+
SDValue Poison = DAG.getPOISON(VT);
4850+
Results.push_back(Poison);
4851+
Results.push_back(Poison);
4852+
}
4853+
48484854
break;
48494855
}
48504856
case ISD::FLOG:
@@ -4934,7 +4940,7 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
49344940
EVT VT = Node->getValueType(0);
49354941
RTLIB::Libcall LC = Node->getOpcode() == ISD::FMODF ? RTLIB::getMODF(VT)
49364942
: RTLIB::getFREXP(VT);
4937-
bool Expanded = DAG.expandMultipleResultFPLibCall(LC, Node, Results,
4943+
bool Expanded = DAG.expandMultipleResultFPLibCall(LC, Node, Results, VT,
49384944
/*CallRetResNo=*/0);
49394945
if (!Expanded)
49404946
llvm_unreachable("Expected scalar FFREXP/FMODF to expand to libcall!");

‎llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1726,7 +1726,8 @@ void DAGTypeLegalizer::ExpandFloatRes_UnaryWithTwoFPResults(
17261726
SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
17271727
assert(!N->isStrictFPOpcode() && "strictfp not implemented");
17281728
SmallVector<SDValue> Results;
1729-
DAG.expandMultipleResultFPLibCall(LC, N, Results, CallRetResNo);
1729+
DAG.expandMultipleResultFPLibCall(LC, N, Results, N->getValueType(0),
1730+
CallRetResNo);
17301731
for (auto [ResNo, Res] : enumerate(Results)) {
17311732
SDValue Lo, Hi;
17321733
GetPairElements(Res, Lo, Hi);

‎llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp‎

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1268,20 +1268,30 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
12681268
return;
12691269

12701270
break;
1271-
case ISD::FSINCOS:
1271+
12721272
case ISD::FSINCOSPI: {
1273+
EVT VT = Node->getValueType(0);
1274+
RTLIB::Libcall LC = RTLIB::getSINCOSPI(VT);
1275+
if (LC != RTLIB::UNKNOWN_LIBCALL &&
1276+
DAG.expandMultipleResultFPLibCall(LC, Node, Results, VT))
1277+
return;
1278+
1279+
// TODO: Try to see if there's a narrower call available to use before
1280+
// scalarizing.
1281+
break;
1282+
}
1283+
case ISD::FSINCOS: {
1284+
// FIXME: Try to directly match vector case like fsincospi
12731285
EVT VT = Node->getValueType(0).getVectorElementType();
1274-
RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
1275-
? RTLIB::getSINCOS(VT)
1276-
: RTLIB::getSINCOSPI(VT);
1277-
if (DAG.expandMultipleResultFPLibCall(LC, Node, Results))
1286+
RTLIB::Libcall LC = RTLIB::getSINCOS(VT);
1287+
if (DAG.expandMultipleResultFPLibCall(LC, Node, Results, VT))
12781288
return;
12791289
break;
12801290
}
12811291
case ISD::FMODF: {
1282-
RTLIB::Libcall LC =
1283-
RTLIB::getMODF(Node->getValueType(0).getVectorElementType());
1284-
if (DAG.expandMultipleResultFPLibCall(LC, Node, Results,
1292+
EVT VT = Node->getValueType(0).getVectorElementType();
1293+
RTLIB::Libcall LC = RTLIB::getMODF(VT);
1294+
if (DAG.expandMultipleResultFPLibCall(LC, Node, Results, VT,
12851295
/*CallRetResNo=*/0))
12861296
return;
12871297
break;

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp‎

Lines changed: 40 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2514,18 +2514,20 @@ static bool canFoldStoreIntoLibCallOutputPointers(StoreSDNode *StoreNode,
25142514

25152515
bool SelectionDAG::expandMultipleResultFPLibCall(
25162516
RTLIB::Libcall LC, SDNode *Node, SmallVectorImpl<SDValue> &Results,
2517-
std::optional<unsigned> CallRetResNo) {
2518-
LLVMContext &Ctx = *getContext();
2519-
EVT VT = Node->getValueType(0);
2520-
unsigned NumResults = Node->getNumValues();
2521-
2517+
EVT CallVT, std::optional<unsigned> CallRetResNo) {
25222518
if (LC == RTLIB::UNKNOWN_LIBCALL)
25232519
return false;
25242520

2525-
const char *LCName = TLI->getLibcallName(LC);
2526-
if (!LCName)
2521+
EVT VT = Node->getValueType(0);
2522+
2523+
RTLIB::LibcallImpl Impl = TLI->getLibcallImpl(LC);
2524+
if (Impl == RTLIB::Unsupported)
25272525
return false;
25282526

2527+
StringRef LCName = TLI->getLibcallImplName(Impl);
2528+
2529+
// FIXME: This should not use TargetLibraryInfo. There should be
2530+
// RTLIB::Libcall entries for each used vector type, and directly matched.
25292531
auto getVecDesc = [&]() -> VecDesc const * {
25302532
for (bool Masked : {false, true}) {
25312533
if (VecDesc const *VD = getLibInfo().getVectorMappingInfo(
@@ -2538,9 +2540,34 @@ bool SelectionDAG::expandMultipleResultFPLibCall(
25382540

25392541
// For vector types, we must find a vector mapping for the libcall.
25402542
VecDesc const *VD = nullptr;
2541-
if (VT.isVector() && !(VD = getVecDesc()))
2543+
if (VT.isVector() && !CallVT.isVector() && !(VD = getVecDesc()))
25422544
return false;
25432545

2546+
bool IsMasked = (VD && VD->isMasked()) ||
2547+
RTLIB::RuntimeLibcallsInfo::hasVectorMaskArgument(Impl);
2548+
2549+
// This wrapper function exists because getVectorMappingInfo works in terms of
2550+
// function names instead of RTLIB enums.
2551+
2552+
// FIXME: If we used a vector mapping, this assumes the calling convention of
2553+
// the vector function is the same as the scalar.
2554+
2555+
StringRef Name = VD ? VD->getVectorFnName() : LCName;
2556+
2557+
return expandMultipleResultFPLibCall(Name,
2558+
TLI->getLibcallImplCallingConv(Impl),
2559+
Node, Results, CallRetResNo, IsMasked);
2560+
}
2561+
2562+
// FIXME: This belongs in TargetLowering
2563+
bool SelectionDAG::expandMultipleResultFPLibCall(
2564+
StringRef Name, CallingConv::ID CC, SDNode *Node,
2565+
SmallVectorImpl<SDValue> &Results, std::optional<unsigned> CallRetResNo,
2566+
bool IsMasked) {
2567+
LLVMContext &Ctx = *getContext();
2568+
EVT VT = Node->getValueType(0);
2569+
unsigned NumResults = Node->getNumValues();
2570+
25442571
// Find users of the node that store the results (and share input chains). The
25452572
// destination pointers can be used instead of creating stack allocations.
25462573
SDValue StoresInChain;
@@ -2598,7 +2625,7 @@ bool SelectionDAG::expandMultipleResultFPLibCall(
25982625
SDLoc DL(Node);
25992626

26002627
// Pass the vector mask (if required).
2601-
if (VD && VD->isMasked()) {
2628+
if (IsMasked) {
26022629
EVT MaskVT = TLI->getSetCCResultType(getDataLayout(), Ctx, VT);
26032630
SDValue Mask = getBoolConstant(true, DL, MaskVT, VT);
26042631
Args.emplace_back(Mask, MaskVT.getTypeForEVT(Ctx));
@@ -2608,11 +2635,11 @@ bool SelectionDAG::expandMultipleResultFPLibCall(
26082635
? Node->getValueType(*CallRetResNo).getTypeForEVT(Ctx)
26092636
: Type::getVoidTy(Ctx);
26102637
SDValue InChain = StoresInChain ? StoresInChain : getEntryNode();
2611-
SDValue Callee = getExternalSymbol(VD ? VD->getVectorFnName().data() : LCName,
2612-
TLI->getPointerTy(getDataLayout()));
2638+
SDValue Callee =
2639+
getExternalSymbol(Name.data(), TLI->getPointerTy(getDataLayout()));
26132640
TargetLowering::CallLoweringInfo CLI(*this);
2614-
CLI.setDebugLoc(DL).setChain(InChain).setLibCallee(
2615-
TLI->getLibcallCallingConv(LC), RetType, Callee, std::move(Args));
2641+
CLI.setDebugLoc(DL).setChain(InChain).setLibCallee(CC, RetType, Callee,
2642+
std::move(Args));
26162643

26172644
auto [Call, CallChain] = TLI->LowerCallTo(CLI);
26182645

‎llvm/lib/Target/X86/X86ISelLowering.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -635,6 +635,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
635635
setOperationAction(ISD::FROUNDEVEN, VT, Action);
636636
setOperationAction(ISD::FTRUNC, VT, Action);
637637
setOperationAction(ISD::FLDEXP, VT, Action);
638+
setOperationAction(ISD::FSINCOSPI, VT, Action);
638639
};
639640

640641
if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) {

‎llvm/test/CodeGen/AArch64/llvm.sincospi.error.ll‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
; RUN: not llc -mtriple=aarch64-gnu-linux -filetype=null %s 2>&1 | FileCheck %s
2+
3+
; CHECK: error: no libcall available for fsincospi
4+
define { float, float } @test_sincospi_f32(float %a) {
5+
%result = call { float, float } @llvm.sincospi.f32(float %a)
6+
ret { float, float } %result
7+
}
8+
9+
; CHECK: error: no libcall available for fsincospi
10+
define { double, double } @test_sincospi_f64(double %a) {
11+
%result = call { double, double } @llvm.sincospi.f64(double %a)
12+
ret { double, double } %result
13+
}

0 commit comments

Comments
 (0)