Skip to content

Commit db90e14

Browse files
krystian-andrzejewskiigcbot
krystian-andrzejewski
authored and
igcbot
committed
Support more conversion intrinsics in PreCompiledFuncImport
This change is to allow the compiler to emulate conversions from fp64 to fp 16 using a non-default rounding mode. Note that the emulation requires a full list of addition operations with a non-default rounding mode. Moreover, there are introduced other IGC intrinsics for completeness only.
1 parent 6399a00 commit db90e14

12 files changed

+6942
-6033
lines changed

‎IGC/Compiler/Builtins/PreCompiledFuncLibrary.hpp‎

Lines changed: 6335 additions & 5928 deletions
Large diffs are not rendered by default.

‎IGC/Compiler/CISACodeGen/EmitVISAPass.cpp‎

Lines changed: 22 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -9166,15 +9166,18 @@ void EmitPass::EmitGenIntrinsicMessage(llvm::GenIntrinsicInst* inst)
91669166
break;
91679167
}
91689168
case GenISAIntrinsic::GenISA_mul_rtz:
9169-
case GenISAIntrinsic::GenISA_fma_rtz:
9169+
case GenISAIntrinsic::GenISA_mul_rte:
9170+
case GenISAIntrinsic::GenISA_mul_rtp:
9171+
case GenISAIntrinsic::GenISA_mul_rtn:
91709172
case GenISAIntrinsic::GenISA_add_rtz:
9171-
emitFPOrtz(inst);
9172-
break;
9173+
case GenISAIntrinsic::GenISA_add_rte:
9174+
case GenISAIntrinsic::GenISA_add_rtp:
9175+
case GenISAIntrinsic::GenISA_add_rtn:
9176+
case GenISAIntrinsic::GenISA_fma_rtz:
9177+
case GenISAIntrinsic::GenISA_fma_rte:
91739178
case GenISAIntrinsic::GenISA_fma_rtp:
9174-
emitFMArtp(inst);
9175-
break;
91769179
case GenISAIntrinsic::GenISA_fma_rtn:
9177-
emitFMArtn(inst);
9180+
emitFPOWithNonDefaultRoundingMode(inst);
91789181
break;
91799182
case GenISAIntrinsic::GenISA_CatchAllDebugLine:
91809183
emitDebugPlaceholder(inst);
@@ -16959,8 +16962,8 @@ void EmitPass::emitfitof(llvm::GenIntrinsicInst* inst)
1695916962
ResetRoundingMode(inst);
1696016963
}
1696116964

16962-
// Emit FP Operations (FPO) using round-to-zero (rtz)
16963-
void EmitPass::emitFPOrtz(llvm::GenIntrinsicInst* inst)
16965+
// Emit FP Operations (FPO) using a non-default rounding mode
16966+
void EmitPass::emitFPOWithNonDefaultRoundingMode(llvm::GenIntrinsicInst* inst)
1696416967
{
1696516968
IGC_ASSERT_MESSAGE(IGCLLVM::getNumArgOperands(inst) >= 2, "ICE: incorrect gen intrinsic");
1696616969

@@ -16969,22 +16972,31 @@ void EmitPass::emitFPOrtz(llvm::GenIntrinsicInst* inst)
1696916972
CVariable* src1 = GetSymbol(inst->getOperand(1));
1697016973
CVariable* dst = m_destination;
1697116974

16972-
SetRoundingMode_FP(ERoundingMode::ROUND_TO_ZERO);
16975+
SetRoundingMode_FP(GetRoundingMode_FP(m_pCtx->getModuleMetaData(), inst));
1697316976

1697416977
switch (GID)
1697516978
{
1697616979
default:
1697716980
IGC_ASSERT_MESSAGE(0, "ICE: unexpected Gen Intrinsic");
1697816981
break;
1697916982
case GenISAIntrinsic::GenISA_mul_rtz:
16983+
case GenISAIntrinsic::GenISA_mul_rte:
16984+
case GenISAIntrinsic::GenISA_mul_rtp:
16985+
case GenISAIntrinsic::GenISA_mul_rtn:
1698016986
m_encoder->Mul(dst, src0, src1);
1698116987
m_encoder->Push();
1698216988
break;
16983-
case GenISAIntrinsic::GenISA_add_rtz:
16989+
case GenISAIntrinsic::GenISA_add_rtz:
16990+
case GenISAIntrinsic::GenISA_add_rte:
16991+
case GenISAIntrinsic::GenISA_add_rtp:
16992+
case GenISAIntrinsic::GenISA_add_rtn:
1698416993
m_encoder->Add(dst, src0, src1);
1698516994
m_encoder->Push();
1698616995
break;
1698716996
case GenISAIntrinsic::GenISA_fma_rtz:
16997+
case GenISAIntrinsic::GenISA_fma_rte:
16998+
case GenISAIntrinsic::GenISA_fma_rtp:
16999+
case GenISAIntrinsic::GenISA_fma_rtn:
1698817000
{
1698917001
CVariable* src2 = GetSymbol(inst->getOperand(2));
1699017002
m_encoder->Mad(dst, src0, src1, src2);
@@ -16996,40 +17008,6 @@ void EmitPass::emitFPOrtz(llvm::GenIntrinsicInst* inst)
1699617008
ResetRoundingMode(inst);
1699717009
}
1699817010

16999-
// Emit FP mad (FMA) using round-to-positive-infinity (rtp)
17000-
void EmitPass::emitFMArtp(llvm::GenIntrinsicInst *inst) {
17001-
IGC_ASSERT_MESSAGE(IGCLLVM::getNumArgOperands(inst) == 3, "ICE: incorrect gen intrinsic");
17002-
17003-
CVariable *src0 = GetSymbol(inst->getOperand(0));
17004-
CVariable *src1 = GetSymbol(inst->getOperand(1));
17005-
CVariable *src2 = GetSymbol(inst->getOperand(2));
17006-
CVariable *dst = m_destination;
17007-
17008-
SetRoundingMode_FP(ERoundingMode::ROUND_TO_POSITIVE);
17009-
17010-
m_encoder->Mad(dst, src0, src1, src2);
17011-
m_encoder->Push();
17012-
17013-
ResetRoundingMode(inst);
17014-
}
17015-
17016-
// Emit FP mad (FMA) using round-to-negative-infinity (rtn)
17017-
void EmitPass::emitFMArtn(llvm::GenIntrinsicInst *inst) {
17018-
IGC_ASSERT_MESSAGE(IGCLLVM::getNumArgOperands(inst) == 3, "ICE: incorrect gen intrinsic");
17019-
17020-
CVariable *src0 = GetSymbol(inst->getOperand(0));
17021-
CVariable *src1 = GetSymbol(inst->getOperand(1));
17022-
CVariable *src2 = GetSymbol(inst->getOperand(2));
17023-
CVariable *dst = m_destination;
17024-
17025-
SetRoundingMode_FP(ERoundingMode::ROUND_TO_NEGATIVE);
17026-
17027-
m_encoder->Mad(dst, src0, src1, src2);
17028-
m_encoder->Push();
17029-
17030-
ResetRoundingMode(inst);
17031-
}
17032-
1703317011
void EmitPass::emitftoi(llvm::GenIntrinsicInst* inst)
1703417012
{
1703517013
IGC_ASSERT_MESSAGE(inst->getOperand(0)->getType()->isFloatingPointTy(), "Unsupported type");

‎IGC/Compiler/CISACodeGen/EmitVISAPass.hpp‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -419,9 +419,7 @@ class EmitPass : public llvm::FunctionPass
419419

420420
void emitf32tof16_rtz(llvm::GenIntrinsicInst* inst);
421421
void emitfitof(llvm::GenIntrinsicInst* inst);
422-
void emitFPOrtz(llvm::GenIntrinsicInst* inst);
423-
void emitFMArtp(llvm::GenIntrinsicInst* inst);
424-
void emitFMArtn(llvm::GenIntrinsicInst* inst);
422+
void emitFPOWithNonDefaultRoundingMode(llvm::GenIntrinsicInst* inst);
425423
void emitftoi(llvm::GenIntrinsicInst* inst);
426424
void emitCtlz(const SSource& source);
427425

‎IGC/Compiler/CISACodeGen/helper.cpp‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2300,16 +2300,23 @@ namespace IGC
23002300
case GenISAIntrinsic::GenISA_ftof_rtn:
23012301
case GenISAIntrinsic::GenISA_itof_rtn:
23022302
case GenISAIntrinsic::GenISA_uitof_rtn:
2303+
case GenISAIntrinsic::GenISA_add_rtn:
2304+
case GenISAIntrinsic::GenISA_mul_rtn:
23032305
case GenISAIntrinsic::GenISA_fma_rtn:
23042306
RM = ERoundingMode::ROUND_TO_NEGATIVE;
23052307
break;
23062308
case GenISAIntrinsic::GenISA_ftof_rtp:
23072309
case GenISAIntrinsic::GenISA_itof_rtp:
23082310
case GenISAIntrinsic::GenISA_uitof_rtp:
2311+
case GenISAIntrinsic::GenISA_add_rtp:
2312+
case GenISAIntrinsic::GenISA_mul_rtp:
23092313
case GenISAIntrinsic::GenISA_fma_rtp:
23102314
RM = ERoundingMode::ROUND_TO_POSITIVE;
23112315
break;
23122316
case GenISAIntrinsic::GenISA_ftof_rte:
2317+
case GenISAIntrinsic::GenISA_add_rte:
2318+
case GenISAIntrinsic::GenISA_mul_rte:
2319+
case GenISAIntrinsic::GenISA_fma_rte:
23132320
RM = ERoundingMode::ROUND_TO_NEAREST_EVEN;
23142321
break;
23152322
case GenISAIntrinsic::GenISA_ftobf:

‎IGC/Compiler/Optimizer/PreCompiledFuncImport.cpp‎

Lines changed: 91 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ SPDX-License-Identifier: MIT
2323
#include "llvm/Linker/Linker.h"
2424
#include "llvm/Support/SourceMgr.h"
2525
#include "llvm/IRReader/IRReader.h"
26+
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
2627
#include "common/LLVMWarningsPop.hpp"
2728
#include "AdaptorCommon/ImplicitArgs.hpp"
2829
#include "AdaptorCommon/AddImplicitArgs.hpp"
@@ -596,6 +597,7 @@ inline bool isPrecompiledEmulationFunction(Function* func)
596597
func->getName().contains("precompiled_u32divrem") ||
597598
func->getName().contains("precompiled_s32divrem_sp") ||
598599
func->getName().contains("precompiled_u32divrem_sp") ||
600+
func->getName().contains("precompiled_convert_f64_to_f16") ||
599601
func->getName().contains("__igcbuiltin_sp_div") ||
600602
func->getName().contains("__igcbuiltin_dp_div_nomadm_ieee");
601603
}
@@ -765,44 +767,29 @@ bool PreCompiledFuncImport::runOnModule(Module& M)
765767
{
766768
createIntrinsicCall(CI, GenISAIntrinsic::GenISA_fma_rtn);
767769
}
768-
}
769-
}
770-
}
771-
}
772-
}
773-
}
774-
775-
//post-process the Int32 precompiled emulation function for div/rem
776-
if (isI32DivRem() || isI32DivRemSP() || isSPDiv())
777-
{
778-
for (auto FI = M.begin(), FE = M.end(); FI != FE; )
779-
{
780-
llvm::Function* func = &(*FI);
781-
++FI;
782-
if (isPrecompiledEmulationFunction(func))
783-
{
784-
for (auto BBI = func->begin(), BBE = func->end(); BBI != BBE; )
785-
{
786-
llvm::BasicBlock* BB = &(*BBI);
787-
++BBI;
788-
for (auto I = BB->begin(), IE = BB->end(); I != IE; I++)
789-
{
790-
if (CallInst * CI = dyn_cast<CallInst>(I))
791-
{
792-
if (Function* calledFunc = CI->getCalledFunction())
770+
else if (calledFunc->getName().startswith("GenISA_add_rte"))
771+
{
772+
createIntrinsicCall(CI, GenISAIntrinsic::GenISA_add_rte);
773+
}
774+
else if (calledFunc->getName().startswith("GenISA_add_rtz"))
775+
{
776+
createIntrinsicCall(CI, GenISAIntrinsic::GenISA_add_rtz);
777+
}
778+
else if (calledFunc->getName().startswith("GenISA_add_rtn"))
779+
{
780+
createIntrinsicCall(CI, GenISAIntrinsic::GenISA_add_rtn);
781+
}
782+
else if (calledFunc->getName().startswith("GenISA_add_rtp"))
783+
{
784+
createIntrinsicCall(CI, GenISAIntrinsic::GenISA_add_rtp);
785+
}
786+
else if (calledFunc->getName().startswith("GenISA_mul_rtz"))
787+
{
788+
createIntrinsicCall(CI, GenISAIntrinsic::GenISA_mul_rtz);
789+
}
790+
else if (calledFunc->getName().startswith("GenISA_uitof_rtz"))
793791
{
794-
if (calledFunc->getName().startswith("GenISA_mul_rtz"))
795-
{
796-
createIntrinsicCall(CI, GenISAIntrinsic::GenISA_mul_rtz);
797-
}
798-
else if (calledFunc->getName().startswith("GenISA_add_rtz"))
799-
{
800-
createIntrinsicCall(CI, GenISAIntrinsic::GenISA_add_rtz);
801-
}
802-
else if (calledFunc->getName().startswith("GenISA_uitof_rtz"))
803-
{
804-
createIntrinsicCall(CI, GenISAIntrinsic::GenISA_uitof_rtz);
805-
}
792+
createIntrinsicCall(CI, GenISAIntrinsic::GenISA_uitof_rtz);
806793
}
807794
}
808795
}
@@ -1525,7 +1512,7 @@ void PreCompiledFuncImport::getInt64DivideEmuType(EmulatedFunctions function, un
15251512
void PreCompiledFuncImport::visitFPTruncInst(llvm::FPTruncInst& inst)
15261513
{
15271514
m_pCtx->metrics.StatBeginEmuFunc(&inst);
1528-
if ((isFP64toFP16() || isDPEmu() || isDPConvEmu()) &&
1515+
if ((isRTEFP64toFP16() || isDPEmu() || isDPConvEmu()) &&
15291516
inst.getDestTy()->isHalfTy() && inst.getSrcTy()->isDoubleTy())
15301517
{
15311518
if (inst.getDestTy()->isVectorTy())
@@ -1534,7 +1521,16 @@ void PreCompiledFuncImport::visitFPTruncInst(llvm::FPTruncInst& inst)
15341521
return;
15351522
}
15361523

1537-
const StringRef funcName = "__precompiled_convert_f64_to_f16";
1524+
ERoundingMode RM = static_cast<ERoundingMode>(m_pCtx->getModuleMetaData()->compOpt.FloatRoundingMode);
1525+
if (isRTEFP64toFP16() && !isDPEmu() && !isDPConvEmu() && RM != ERoundingMode::ROUND_TO_NEAREST_EVEN)
1526+
{
1527+
// It is safe to use a sequence of conversions for RTZ, RTN and RTP (fp64 -> fp32 -> fp16)
1528+
return;
1529+
}
1530+
const StringRef funcName = RM == ERoundingMode::ROUND_TO_ZERO ? "__precompiled_convert_f64_to_f16_rtz" :
1531+
RM == ERoundingMode::ROUND_TO_NEGATIVE ? "__precompiled_convert_f64_to_f16_rtn" :
1532+
RM == ERoundingMode::ROUND_TO_POSITIVE ? "__precompiled_convert_f64_to_f16_rtp" :
1533+
"__precompiled_convert_f64_to_f16_rte";
15381534
Function* func = m_pModule->getFunction(funcName);
15391535

15401536
// Try to look up the function in the module's symbol
@@ -1553,12 +1549,9 @@ void PreCompiledFuncImport::visitFPTruncInst(llvm::FPTruncInst& inst)
15531549
m_pModule);
15541550
}
15551551

1556-
CallInst* funcCall = CallInst::Create(func, inst.getOperand(0), inst.getName(), &inst);
1552+
CallInst* funcCall = CallInst::Create(func, inst.getOperand(0));
15571553
addCallInst(funcCall);
1558-
funcCall->setDebugLoc(inst.getDebugLoc());
1559-
1560-
inst.replaceAllUsesWith(funcCall);
1561-
inst.eraseFromParent();
1554+
ReplaceInstWithInst(&inst, funcCall);
15621555

15631556
m_libModuleToBeImported[LIBMOD_INT_DIV_REM] = true;
15641557
m_changed = true;
@@ -2307,11 +2300,8 @@ As a result, we reduce 2x necessary work
23072300
m_CallRemDiv.push_back(&I);
23082301
}
23092302

2310-
if (!isDPEmu() && !isDPDivSqrtEmu()) {
2311-
return;
2312-
}
2313-
2314-
if (resTy->isDoubleTy() &&
2303+
if ((isDPEmu() || isDPDivSqrtEmu()) &&
2304+
resTy->isDoubleTy() &&
23152305
(II && II->getIntrinsicID() == Intrinsic::sqrt))
23162306
{
23172307
FunctionIDs sqrtType = FUNCTION_DP_SQRT;
@@ -2346,12 +2336,8 @@ As a result, we reduce 2x necessary work
23462336
return;
23472337
}
23482338

2349-
if (!isDPEmu()) {
2350-
return;
2351-
}
2352-
23532339
// llvm.fma.f64
2354-
if (resTy->isDoubleTy() && II && II->getIntrinsicID() == Intrinsic::fma)
2340+
if (isDPEmu() && resTy->isDoubleTy() && II && II->getIntrinsicID() == Intrinsic::fma)
23552341
{
23562342
Function* newFunc = getOrCreateFunction(FUNCTION_DP_FMA);
23572343
Function* CurrFunc = I.getParent()->getParent();
@@ -2378,7 +2364,7 @@ As a result, we reduce 2x necessary work
23782364
// llvm.fma.rtn.f64
23792365
// llvm.fma.rtp.f64
23802366
// llvm.fma.rtz.f64
2381-
if (resTy->isDoubleTy() && GII &&
2367+
if (isDPEmu() && resTy->isDoubleTy() && GII &&
23822368
(GII->getIntrinsicID() == GenISAIntrinsic::GenISA_fma_rtn ||
23832369
GII->getIntrinsicID() == GenISAIntrinsic::GenISA_fma_rtp ||
23842370
GII->getIntrinsicID() == GenISAIntrinsic::GenISA_fma_rtz))
@@ -2421,8 +2407,57 @@ As a result, we reduce 2x necessary work
24212407
return;
24222408
}
24232409

2410+
if ((isRTEFP64toFP16() || isDPEmu() || isDPConvEmu()) &&
2411+
GII &&
2412+
(GII->getIntrinsicID() == GenISAIntrinsic::GenISA_ftof_rte ||
2413+
GII->getIntrinsicID() == GenISAIntrinsic::GenISA_ftof_rtz ||
2414+
GII->getIntrinsicID() == GenISAIntrinsic::GenISA_ftof_rtn ||
2415+
GII->getIntrinsicID() == GenISAIntrinsic::GenISA_ftof_rtp) &&
2416+
resTy->isHalfTy() && GII->getOperand(0)->getType()->isDoubleTy())
2417+
{
2418+
if (isRTEFP64toFP16() && !isDPEmu() && !isDPConvEmu() && GII->getIntrinsicID() != GenISAIntrinsic::GenISA_ftof_rte)
2419+
{
2420+
// It is safe to use a sequence of conversions for RTZ, RTN and RTP (fp64 -> fp32 -> fp16)
2421+
return;
2422+
}
2423+
const StringRef funcName =
2424+
GII->getIntrinsicID() == GenISAIntrinsic::GenISA_ftof_rte ?
2425+
"__precompiled_convert_f64_to_f16_rte" :
2426+
GII->getIntrinsicID() == GenISAIntrinsic::GenISA_ftof_rtz ?
2427+
"__precompiled_convert_f64_to_f16_rtz" :
2428+
GII->getIntrinsicID() == GenISAIntrinsic::GenISA_ftof_rtn ?
2429+
"__precompiled_convert_f64_to_f16_rtn" :
2430+
"__precompiled_convert_f64_to_f16_rtp";
2431+
Function* func = m_pModule->getFunction(funcName);
2432+
2433+
// Try to look up the function in the module's symbol
2434+
// table first, else add it.
2435+
if (func == NULL)
2436+
{
2437+
FunctionType* FuncIntrType = FunctionType::get(
2438+
resTy,
2439+
I.getOperand(0)->getType(),
2440+
false);
2441+
2442+
func = Function::Create(
2443+
FuncIntrType,
2444+
GlobalValue::ExternalLinkage,
2445+
funcName,
2446+
m_pModule);
2447+
}
2448+
2449+
CallInst* funcCall = CallInst::Create(func, GII->getOperand(0));
2450+
ReplaceInstWithInst(GII, funcCall);
2451+
addCallInst(funcCall);
2452+
2453+
m_libModuleToBeImported[LIBMOD_INT_DIV_REM] = true;
2454+
m_changed = true;
2455+
m_pCtx->metrics.StatEndEmuFunc(funcCall);
2456+
return;
2457+
}
2458+
24242459
// llvm.fabs.f64
2425-
if (resTy->isDoubleTy() && II && II->getIntrinsicID() == Intrinsic::fabs)
2460+
if (isDPEmu() && resTy->isDoubleTy() && II && II->getIntrinsicID() == Intrinsic::fabs)
24262461
{
24272462
// bit 63 is sign bit, set it to zero. Don't use int64.
24282463
VectorType* vec2Ty = IGCLLVM::FixedVectorType::get(intTy, 2);

‎IGC/Compiler/Optimizer/PreCompiledFuncImport.hpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ namespace IGC
4040
EMU_SP_DIV = 0x10, // bit 4: IEEE-complaint float div emulation (float)
4141
EMU_I32DIVREM = 0x20, // bit 5: emulation lib for i32 div/rem
4242
EMU_I32DIVREM_SP = 0x40, // bit 6: emulation lib for i32 div/rem using fp32
43-
EMU_FP64_FP16_CONV = 0x80 // bit 7: fp64 to fp32 conversion
43+
EMU_FP64_FP16_CONV = 0x80 // bit 7: fp64 to fp32 rte conversion
4444
};
4545

4646
class PreCompiledFuncImport : public llvm::ModulePass, public llvm::InstVisitor<PreCompiledFuncImport>
@@ -284,7 +284,7 @@ namespace IGC
284284
bool isI32DivRem() const { return (m_emuKind & EmuKind::EMU_I32DIVREM) > 0; }
285285
bool isI32DivRemSP() const { return (m_emuKind & EmuKind::EMU_I32DIVREM_SP) > 0; }
286286
bool isSPDiv() const { return (m_emuKind & EmuKind::EMU_SP_DIV) > 0; }
287-
bool isFP64toFP16() const { return (m_emuKind & EmuKind::EMU_FP64_FP16_CONV) > 0; }
287+
bool isRTEFP64toFP16() const { return (m_emuKind & EmuKind::EMU_FP64_FP16_CONV) > 0; }
288288

289289
bool m_libModuleToBeImported[NUM_LIBMODS];
290290
bool m_libModuleAlreadyImported[NUM_LIBMODS];

0 commit comments

Comments
 (0)