From 7d6a5d3f29864fc3cb3ff05a8ff8ff47251f22cf Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Thu, 12 Dec 2024 09:47:18 -0800 Subject: [PATCH 1/6] introduce SPV_INTEL_float_controls2; add arith Constrained Floating-Point Intrinsics; add float rounding mode decoration --- llvm/docs/SPIRVUsage.rst | 2 + llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp | 10 ++- llvm/lib/Target/SPIRV/SPIRVBuiltins.h | 3 +- llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp | 2 + llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 79 ++++++++++++++++- .../Target/SPIRV/SPIRVInstructionSelector.cpp | 11 +++ llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp | 8 ++ llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp | 25 +++++- .../lib/Target/SPIRV/SPIRVSymbolicOperands.td | 8 ++ .../exec_mode_float_control_empty.ll | 18 ++++ .../exec_mode_float_control_intel.ll | 74 ++++++++++++++++ .../SPIRV/instructions/integer-casts.ll | 8 ++ .../llvm-intrinsics/constrained-arithmetic.ll | 84 +++++++++++++++++++ 13 files changed, 323 insertions(+), 9 deletions(-) create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_float_controls2/exec_mode_float_control_empty.ll create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_float_controls2/exec_mode_float_control_intel.ll create mode 100644 llvm/test/CodeGen/SPIRV/llvm-intrinsics/constrained-arithmetic.ll diff --git a/llvm/docs/SPIRVUsage.rst b/llvm/docs/SPIRVUsage.rst index 8f7ac71f8026b..b7b3d21545168 100644 --- a/llvm/docs/SPIRVUsage.rst +++ b/llvm/docs/SPIRVUsage.rst @@ -159,6 +159,8 @@ list of supported SPIR-V extensions, sorted alphabetically by their extension na - Adds instructions to convert between single-precision 32-bit floating-point values and 16-bit bfloat16 values. * - ``SPV_INTEL_cache_controls`` - Allows cache control information to be applied to memory access instructions. + * - ``SPV_INTEL_float_controls2`` + - Adds execution modes and decorations to control floating-point computations. * - ``SPV_INTEL_function_pointers`` - Allows translation of function pointers. * - ``SPV_INTEL_inline_assembly`` diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp index f4bfda4932b16..806eadffca6e7 100644 --- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp @@ -173,7 +173,8 @@ using namespace InstructionSet; namespace SPIRV { /// Parses the name part of the demangled builtin call. -std::string lookupBuiltinNameHelper(StringRef DemangledCall) { +std::string lookupBuiltinNameHelper(StringRef DemangledCall, + std::string *Postfix) { const static std::string PassPrefix = "(anonymous namespace)::"; std::string BuiltinName; // Itanium Demangler result may have "(anonymous namespace)::" prefix @@ -231,10 +232,13 @@ std::string lookupBuiltinNameHelper(StringRef DemangledCall) { "ReadClockKHR|SubgroupBlockReadINTEL|SubgroupImageBlockReadINTEL|" "SubgroupImageMediaBlockReadINTEL|SubgroupImageMediaBlockWriteINTEL|" "Convert|" - "UConvert|SConvert|FConvert|SatConvert).*)_R.*"); + "UConvert|SConvert|FConvert|SatConvert).*)_R(.*)"); std::smatch Match; - if (std::regex_match(BuiltinName, Match, SpvWithR) && Match.size() > 2) + if (std::regex_match(BuiltinName, Match, SpvWithR) && Match.size() > 3) { BuiltinName = Match[1].str(); + if (Postfix) + *Postfix = Match[3].str(); + } return BuiltinName; } diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.h b/llvm/lib/Target/SPIRV/SPIRVBuiltins.h index 42b452db8b9fb..0182d9652d18c 100644 --- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.h +++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.h @@ -20,7 +20,8 @@ namespace llvm { namespace SPIRV { /// Parses the name part of the demangled builtin call. -std::string lookupBuiltinNameHelper(StringRef DemangledCall); +std::string lookupBuiltinNameHelper(StringRef DemangledCall, + std::string *Postfix = nullptr); /// Lowers a builtin function call using the provided \p DemangledCall skeleton /// and external instruction \p Set. /// diff --git a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp index fb05c1fdbd1e3..45b39c5116479 100644 --- a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp @@ -36,6 +36,8 @@ static const std::map> SPIRV::Extension::Extension::SPV_INTEL_arbitrary_precision_integers}, {"SPV_INTEL_cache_controls", SPIRV::Extension::Extension::SPV_INTEL_cache_controls}, + {"SPV_INTEL_float_controls2", + SPIRV::Extension::Extension::SPV_INTEL_float_controls2}, {"SPV_INTEL_global_variable_fpga_decorations", SPIRV::Extension::Extension:: SPV_INTEL_global_variable_fpga_decorations}, diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 2b623136e602e..5ab3810900033 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -216,6 +216,8 @@ class SPIRVEmitIntrinsics bool processFunctionPointers(Module &M); void parseFunDeclarations(Module &M); + void useRoundingMode(ConstrainedFPIntrinsic *FPI, IRBuilder<> &B); + public: static char ID; SPIRVEmitIntrinsics() : ModulePass(ID) { @@ -1291,6 +1293,21 @@ void SPIRVEmitIntrinsics::preprocessCompositeConstants(IRBuilder<> &B) { } } +static void createRoundingModeDecoration(Instruction *I, + unsigned RoundingModeDeco, + IRBuilder<> &B) { + LLVMContext &Ctx = I->getContext(); + Type *Int32Ty = Type::getInt32Ty(Ctx); + SmallVector MDs = { + MDNode::get(Ctx, ConstantAsMetadata::get(ConstantInt::get( + Int32Ty, SPIRV::Decoration::FPRoundingMode))), + MDNode::get(Ctx, ConstantAsMetadata::get( + ConstantInt::get(Int32Ty, RoundingModeDeco)))}; + setInsertPointAfterDef(B, I); + B.CreateIntrinsic(Intrinsic::spv_assign_decoration, {I->getType()}, + {I, MetadataAsValue::get(Ctx, MDNode::get(Ctx, MDs))}); +} + Instruction *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) { if (!Call.isInlineAsm()) return &Call; @@ -1312,6 +1329,40 @@ Instruction *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) { return &Call; } +// Use a tip about rounding mode to create a decoration. +void SPIRVEmitIntrinsics::useRoundingMode(ConstrainedFPIntrinsic *FPI, + IRBuilder<> &B) { + std::optional RM = FPI->getRoundingMode(); + if (!RM.has_value()) + return; + unsigned RoundingModeDeco = std::numeric_limits::max(); + switch (RM.value()) { + default: + // ignore unknown rounding modes + break; + case RoundingMode::NearestTiesToEven: + RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTE; + break; + case RoundingMode::TowardNegative: + RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTN; + break; + case RoundingMode::TowardPositive: + RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTP; + break; + case RoundingMode::TowardZero: + RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTZ; + break; + case RoundingMode::Dynamic: + case RoundingMode::NearestTiesToAway: + // TODO: check if supported + break; + } + if (RoundingModeDeco == std::numeric_limits::max()) + return; + // Convert the tip about rounding mode into a decoration record. + createRoundingModeDecoration(FPI, RoundingModeDeco, B); +} + Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) { BasicBlock *ParentBB = I.getParent(); IRBuilder<> B(ParentBB); @@ -1809,6 +1860,18 @@ bool SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(Instruction *I, return true; } +static unsigned roundingModeMDToDecorationConst(StringRef S) { + if (S == "rte") + return SPIRV::FPRoundingMode::FPRoundingMode::RTE; + if (S == "rtz") + return SPIRV::FPRoundingMode::FPRoundingMode::RTZ; + if (S == "rtp") + return SPIRV::FPRoundingMode::FPRoundingMode::RTP; + if (S == "rtn") + return SPIRV::FPRoundingMode::FPRoundingMode::RTN; + return std::numeric_limits::max(); +} + void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I, IRBuilder<> &B) { // TODO: extend the list of functions with known result types @@ -1826,8 +1889,9 @@ void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I, Function *CalledF = CI->getCalledFunction(); std::string DemangledName = getOclOrSpirvBuiltinDemangledName(CalledF->getName()); + std::string Postfix; if (DemangledName.length() > 0) - DemangledName = SPIRV::lookupBuiltinNameHelper(DemangledName); + DemangledName = SPIRV::lookupBuiltinNameHelper(DemangledName, &Postfix); auto ResIt = ResTypeWellKnown.find(DemangledName); if (ResIt != ResTypeWellKnown.end()) { IsKnown = true; @@ -1839,6 +1903,16 @@ void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I, break; } } + // check if a floating rounding mode info is present + StringRef S = Postfix; + SmallVector Parts; + S.split(Parts, "_", -1, false); + if (Parts.size() > 1) { + // Convert the tip about rounding mode into a decoration record. + unsigned RoundingModeDeco = roundingModeMDToDecorationConst(Parts[1]); + if (RoundingModeDeco != std::numeric_limits::max()) + createRoundingModeDecoration(CI, RoundingModeDeco, B); + } } } @@ -2264,6 +2338,9 @@ bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) { // already, and force it to be i8 if not if (Postpone && !GR->findAssignPtrTypeInstr(I)) insertAssignPtrTypeIntrs(I, B, true); + + if (auto *FPI = dyn_cast(I)) + useRoundingMode(FPI, B); } // Pass backward: use instructions results to specify/update/cast operands diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index b64030508cfc1..690b6b57211bb 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -560,6 +560,17 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg, case TargetOpcode::G_FMA: return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); + case TargetOpcode::G_STRICT_FSQRT: + case TargetOpcode::G_STRICT_FADD: + case TargetOpcode::G_STRICT_FSUB: + case TargetOpcode::G_STRICT_FMUL: + case TargetOpcode::G_STRICT_FDIV: + case TargetOpcode::G_STRICT_FREM: + case TargetOpcode::G_STRICT_FLDEXP: + return false; + case TargetOpcode::G_STRICT_FMA: + return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); + case TargetOpcode::G_FPOW: return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); case TargetOpcode::G_FPOWI: diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp index 7230e0e6b9fca..583a95150ef0c 100644 --- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp @@ -221,6 +221,14 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) { getActionDefinitionsBuilder(G_FMA).legalFor(allFloatScalarsAndVectors); + getActionDefinitionsBuilder({G_STRICT_FSQRT, G_STRICT_FADD, G_STRICT_FSUB, G_STRICT_FMUL, + G_STRICT_FDIV, G_STRICT_FREM, G_STRICT_FMA}) + .legalFor(allFloatScalarsAndVectors); + + getActionDefinitionsBuilder(G_STRICT_FLDEXP) + .legalForCartesianProduct(allFloatScalarsAndVectors, + allFloatScalarsAndVectors); + getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI}) .legalForCartesianProduct(allIntScalarsAndVectors, allFloatScalarsAndVectors); diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp index 4dea4056799fe..6371c67d92458 100644 --- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp @@ -1626,9 +1626,10 @@ static void collectReqs(const Module &M, SPIRV::ModuleAnalysisInfo &MAI, // Collect requirements for OpExecutionMode instructions. auto Node = M.getNamedMetadata("spirv.ExecutionMode"); if (Node) { - // SPV_KHR_float_controls is not available until v1.4 - bool RequireFloatControls = false, + bool RequireFloatControls = false, RequireFloatControls2 = false, VerLower14 = !ST.isAtLeastSPIRVVer(VersionTuple(1, 4)); + bool HasFloatControls2 = + ST.canUseExtension(SPIRV::Extension::SPV_INTEL_float_controls2); for (unsigned i = 0; i < Node->getNumOperands(); i++) { MDNode *MDN = cast(Node->getOperand(i)); const MDOperand &MDOp = MDN->getOperand(1); @@ -1636,8 +1637,7 @@ static void collectReqs(const Module &M, SPIRV::ModuleAnalysisInfo &MAI, Constant *C = CMeta->getValue(); if (ConstantInt *Const = dyn_cast(C)) { auto EM = Const->getZExtValue(); - MAI.Reqs.getAndAddRequirements( - SPIRV::OperandCategory::ExecutionModeOperand, EM, ST); + // SPV_KHR_float_controls is not available until v1.4: // add SPV_KHR_float_controls if the version is too low switch (EM) { case SPIRV::ExecutionMode::DenormPreserve: @@ -1646,7 +1646,22 @@ static void collectReqs(const Module &M, SPIRV::ModuleAnalysisInfo &MAI, case SPIRV::ExecutionMode::RoundingModeRTE: case SPIRV::ExecutionMode::RoundingModeRTZ: RequireFloatControls = VerLower14; + MAI.Reqs.getAndAddRequirements( + SPIRV::OperandCategory::ExecutionModeOperand, EM, ST); break; + case SPIRV::ExecutionMode::RoundingModeRTPINTEL: + case SPIRV::ExecutionMode::RoundingModeRTNINTEL: + case SPIRV::ExecutionMode::FloatingPointModeALTINTEL: + case SPIRV::ExecutionMode::FloatingPointModeIEEEINTEL: + if (HasFloatControls2) { + RequireFloatControls2 = true; + MAI.Reqs.getAndAddRequirements( + SPIRV::OperandCategory::ExecutionModeOperand, EM, ST); + } + break; + default: + MAI.Reqs.getAndAddRequirements( + SPIRV::OperandCategory::ExecutionModeOperand, EM, ST); } } } @@ -1654,6 +1669,8 @@ static void collectReqs(const Module &M, SPIRV::ModuleAnalysisInfo &MAI, if (RequireFloatControls && ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls)) MAI.Reqs.addExtension(SPIRV::Extension::SPV_KHR_float_controls); + if (RequireFloatControls2) + MAI.Reqs.addExtension(SPIRV::Extension::SPV_INTEL_float_controls2); } for (auto FI = M.begin(), E = M.end(); FI != E; ++FI) { const Function &F = *FI; diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td index 745d1e1aec67a..3071df0e28c56 100644 --- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td +++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td @@ -308,6 +308,7 @@ defm SPV_KHR_cooperative_matrix : ExtensionOperand<111>; defm SPV_EXT_arithmetic_fence : ExtensionOperand<112>; defm SPV_EXT_optnone : ExtensionOperand<113>; defm SPV_INTEL_joint_matrix : ExtensionOperand<114>; +defm SPV_INTEL_float_controls2 : ExtensionOperand<115>; //===----------------------------------------------------------------------===// // Multiclass used to define Capabilities enum values and at the same time @@ -501,6 +502,9 @@ defm PackedCooperativeMatrixINTEL : CapabilityOperand<6434, 0, 0, [SPV_INTEL_joi defm CooperativeMatrixInvocationInstructionsINTEL : CapabilityOperand<6435, 0, 0, [SPV_INTEL_joint_matrix], []>; defm CooperativeMatrixTF32ComponentTypeINTEL : CapabilityOperand<6436, 0, 0, [SPV_INTEL_joint_matrix], []>; defm CooperativeMatrixBFloat16ComponentTypeINTEL : CapabilityOperand<6437, 0, 0, [SPV_INTEL_joint_matrix], []>; +defm RoundToInfinityINTEL : CapabilityOperand<5582, 0, 0, [SPV_INTEL_float_controls2], []>; +defm FloatingPointModeINTEL : CapabilityOperand<5583, 0, 0, [SPV_INTEL_float_controls2], []>; +defm FunctionFloatControlINTEL : CapabilityOperand<5821, 0, 0, [SPV_INTEL_float_controls2], []>; //===----------------------------------------------------------------------===// // Multiclass used to define SourceLanguage enum values and at the same time @@ -694,6 +698,10 @@ defm OutputLinesNV : ExecutionModeOperand<5269, [MeshShadingNV]>; defm DerivativeGroupQuadsNV : ExecutionModeOperand<5289, [ComputeDerivativeGroupQuadsNV]>; defm DerivativeGroupLinearNV : ExecutionModeOperand<5290, [ComputeDerivativeGroupLinearNV]>; defm OutputTrianglesNV : ExecutionModeOperand<5298, [MeshShadingNV]>; +defm RoundingModeRTPINTEL : ExecutionModeOperand<5620, [RoundToInfinityINTEL]>; +defm RoundingModeRTNINTEL : ExecutionModeOperand<5621, [RoundToInfinityINTEL]>; +defm FloatingPointModeALTINTEL : ExecutionModeOperand<5622, [FloatingPointModeINTEL]>; +defm FloatingPointModeIEEEINTEL : ExecutionModeOperand<5623, [FloatingPointModeINTEL]>; //===----------------------------------------------------------------------===// // Multiclass used to define StorageClass enum values and at the same time diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_float_controls2/exec_mode_float_control_empty.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_float_controls2/exec_mode_float_control_empty.ll new file mode 100644 index 0000000000000..dca777a150a5b --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_float_controls2/exec_mode_float_control_empty.ll @@ -0,0 +1,18 @@ +; Adapted from https://github.com/KhronosGroup/SPIRV-LLVM-Translator/tree/main/test/extensions/INTEL/SPV_INTEL_float_controls2 + +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_float_controls,+SPV_INTEL_float_controls2 %s -o - | FileCheck %s +; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_float_controls,+SPV_INTEL_float_controls2 %s -o - -filetype=obj | spirv-val %} + +; CHECK-NOT: {{ExecutionMode.*(DenormPreserve|DenormFlushToZero|SignedZeroInfNanPreserve|RoundingModeRTE|RoundingModeRTZ|RoundingModeRTPINTEL|RoundingModeRTNINTEL|FloatingPointModeALTINTEL|FloatingPointModeIEEEINTEL)}} +define dso_local dllexport spir_kernel void @k_no_fc(i32 %ibuf, i32 %obuf) local_unnamed_addr #16 { +entry: + ret void +} + +attributes #16 = { noinline norecurse nounwind readnone "VCMain" "VCFunction" "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 8.0.1"} diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_float_controls2/exec_mode_float_control_intel.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_float_controls2/exec_mode_float_control_intel.ll new file mode 100644 index 0000000000000..5de154053da83 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_float_controls2/exec_mode_float_control_intel.ll @@ -0,0 +1,74 @@ +; Adapted from https://github.com/KhronosGroup/SPIRV-LLVM-Translator/tree/main/test/extensions/INTEL/SPV_INTEL_float_controls2 + +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_float_controls2 %s -o - | FileCheck %s +; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_float_controls2 %s -o - -filetype=obj | spirv-val %} + +; CHECK-DAG: Capability RoundToInfinityINTEL +; CHECK-DAG: Capability FloatingPointModeINTEL +; CHECK: Extension "SPV_INTEL_float_controls2" + +define dso_local dllexport spir_kernel void @k_float_controls_0(i32 %ibuf, i32 %obuf) { +entry: + ret void +} + +define dso_local dllexport spir_kernel void @k_float_controls_1(i32 %ibuf, i32 %obuf) { +entry: + ret void +} + +define dso_local dllexport spir_kernel void @k_float_controls_2(i32 %ibuf, i32 %obuf) { +entry: + ret void +} + +define dso_local dllexport spir_kernel void @k_float_controls_3(i32 %ibuf, i32 %obuf) { +entry: + ret void +} + +!llvm.module.flags = !{!12} +!llvm.ident = !{!13} +!spirv.EntryPoint = !{} +!spirv.ExecutionMode = !{!15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26} + +; CHECK-DAG: OpEntryPoint Kernel %[[#KERNEL0:]] "k_float_controls_0" +; CHECK-DAG: OpEntryPoint Kernel %[[#KERNEL1:]] "k_float_controls_1" +; CHECK-DAG: OpEntryPoint Kernel %[[#KERNEL2:]] "k_float_controls_2" +; CHECK-DAG: OpEntryPoint Kernel %[[#KERNEL3:]] "k_float_controls_3" +!0 = !{ptr @k_float_controls_0, !"k_float_controls_0", !1, i32 0, !2, !3, !4, i32 0, i32 0} +!1 = !{i32 2, i32 2} +!2 = !{i32 32, i32 36} +!3 = !{i32 0, i32 0} +!4 = !{!"", !""} +!12 = !{i32 1, !"wchar_size", i32 4} +!13 = !{!"clang version 8.0.1"} +!14 = !{i32 1, i32 0} + +; CHECK-DAG: OpExecutionMode %[[#KERNEL0]] RoundingModeRTPINTEL 64 +!15 = !{ptr @k_float_controls_0, i32 5620, i32 64} +; CHECK-DAG: OpExecutionMode %[[#KERNEL0]] RoundingModeRTPINTEL 32 +!16 = !{ptr @k_float_controls_0, i32 5620, i32 32} +; CHECK-DAG: OpExecutionMode %[[#KERNEL0]] RoundingModeRTPINTEL 16 +!17 = !{ptr @k_float_controls_0, i32 5620, i32 16} + +; CHECK-DAG: OpExecutionMode %[[#KERNEL1]] RoundingModeRTNINTEL 64 +!18 = !{ptr @k_float_controls_1, i32 5621, i32 64} +; CHECK-DAG: OpExecutionMode %[[#KERNEL1]] RoundingModeRTNINTEL 32 +!19 = !{ptr @k_float_controls_1, i32 5621, i32 32} +; CHECK-DAG: OpExecutionMode %[[#KERNEL1]] RoundingModeRTNINTEL 16 +!20 = !{ptr @k_float_controls_1, i32 5621, i32 16} + +; CHECK-DAG: OpExecutionMode %[[#KERNEL2]] FloatingPointModeALTINTEL 64 +!21 = !{ptr @k_float_controls_2, i32 5622, i32 64} +; CHECK-DAG: OpExecutionMode %[[#KERNEL2]] FloatingPointModeALTINTEL 32 +!22 = !{ptr @k_float_controls_2, i32 5622, i32 32} +; CHECK-DAG: OpExecutionMode %[[#KERNEL2]] FloatingPointModeALTINTEL 16 +!23 = !{ptr @k_float_controls_2, i32 5622, i32 16} + +; CHECK-DAG: OpExecutionMode %[[#KERNEL3]] FloatingPointModeIEEEINTEL 64 +!24 = !{ptr @k_float_controls_3, i32 5623, i32 64} +; CHECK-DAG: OpExecutionMode %[[#KERNEL3]] FloatingPointModeIEEEINTEL 32 +!25 = !{ptr @k_float_controls_3, i32 5623, i32 32} +; CHECK-DAG: OpExecutionMode %[[#KERNEL3]] FloatingPointModeIEEEINTEL 16 +!26 = !{ptr @k_float_controls_3, i32 5623, i32 16} diff --git a/llvm/test/CodeGen/SPIRV/instructions/integer-casts.ll b/llvm/test/CodeGen/SPIRV/instructions/integer-casts.ll index 640dc273dfa62..6763c95b51bf5 100644 --- a/llvm/test/CodeGen/SPIRV/instructions/integer-casts.ll +++ b/llvm/test/CodeGen/SPIRV/instructions/integer-casts.ll @@ -272,6 +272,10 @@ define dso_local spir_kernel void @test_wrappers(ptr addrspace(4) %arg, i64 %arg %r14 = call spir_func <4 x i32> @_Z22__spirv_SConvert_Rint2Dv2_a(<4 x i8> %arg_v2) %r15 = call spir_func float @_Z30__spirv_ConvertUToF_Rfloat_rtz(i64 %arg_ptr) %r16 = call spir_func float @__spirv_ConvertUToF_Rfloat_rtz(i64 %arg_ptr) + %r17 = call spir_func <2 x float> @_Z28__spirv_FConvert_Rfloat2_rtzDv2_DF16_(<2 x half> noundef ) + %r18 = call spir_func <2 x float> @_Z28__spirv_FConvert_Rfloat2_rteDv2_DF16_(<2 x half> noundef ) + %r19 = call spir_func <2 x float> @_Z28__spirv_FConvert_Rfloat2_rtpDv2_DF16_(<2 x half> noundef ) + %r20 = call spir_func <2 x float> @_Z28__spirv_FConvert_Rfloat2_rtnDv2_DF16_(<2 x half> noundef ) ret void } @@ -291,3 +295,7 @@ declare dso_local spir_func <4 x i32> @_Z22__spirv_UConvert_Rint2Dv2_a(<4 x i8>) declare dso_local spir_func <4 x i32> @_Z22__spirv_SConvert_Rint2Dv2_a(<4 x i8>) declare dso_local spir_func float @_Z30__spirv_ConvertUToF_Rfloat_rtz(i64) declare dso_local spir_func float @__spirv_ConvertUToF_Rfloat_rtz(i64) +declare dso_local spir_func <2 x float> @_Z28__spirv_FConvert_Rfloat2_rtzDv2_DF16_(<2 x half> noundef) +declare dso_local spir_func <2 x float> @_Z28__spirv_FConvert_Rfloat2_rteDv2_DF16_(<2 x half> noundef) +declare dso_local spir_func <2 x float> @_Z28__spirv_FConvert_Rfloat2_rtpDv2_DF16_(<2 x half> noundef) +declare dso_local spir_func <2 x float> @_Z28__spirv_FConvert_Rfloat2_rtnDv2_DF16_(<2 x half> noundef) diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/constrained-arithmetic.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/constrained-arithmetic.ll new file mode 100644 index 0000000000000..e30a2fef1d8cd --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/constrained-arithmetic.ll @@ -0,0 +1,84 @@ +; Adapted from https://github.com/KhronosGroup/SPIRV-LLVM-Translator/tree/main/test/llvm-intrinsics + +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK: OpName %[[#ad:]] "add" +; CHECK: OpName %[[#di:]] "div" +; CHECK: OpName %[[#su:]] "sub" +; CHECK: OpName %[[#mu:]] "mul" + +; CHECK-NOT: OpDecorate %[[#]] FPRoundingMode + +; CHECK-DAG: OpDecorate %[[#ad]] FPRoundingMode 0 +; CHECK-DAG: OpDecorate %[[#di]] FPRoundingMode 1 +; CHECK-DAG: OpDecorate %[[#su]] FPRoundingMode 2 +; CHECK-DAG: OpDecorate %[[#mu]] FPRoundingMode 3 + +; CHECK-NOT: OpDecorate %[[#]] FPRoundingMode + +; CHECK: OpFAdd %[[#]] %[[#ad]] +; CHECK: OpFDiv %[[#]] %[[#di]] +; CHECK: OpFSub %[[#]] %[[#su]] +; CHECK: OpFMul %[[#]] %[[#mu]] +; CHECK: OpFMul +; CHECK: OpFAdd +; CHECK: OpExtInst %[[#]] %[[#]] %[[#]] fma +; CHECK: OpFRem + +; Function Attrs: norecurse nounwind strictfp +define dso_local spir_kernel void @test(float %a, i32 %in, i32 %ui) local_unnamed_addr #0 !kernel_arg_addr_space !5 !kernel_arg_access_qual !6 !kernel_arg_type !7 !kernel_arg_base_type !7 !kernel_arg_type_qual !8 !kernel_arg_buffer_location !9 { +entry: + %add = tail call float @llvm.experimental.constrained.fadd.f32(float %a, float %a, metadata !"round.tonearest", metadata !"fpexcept.strict") #2 + %div = tail call float @llvm.experimental.constrained.fdiv.f32(float %add, float %add, metadata !"round.towardzero", metadata !"fpexcept.strict") #2, !fpmath !10 + %sub = tail call float @llvm.experimental.constrained.fsub.f32(float %div, float %div, metadata !"round.upward", metadata !"fpexcept.strict") #2 + %mul = tail call float @llvm.experimental.constrained.fmul.f32(float %sub, float %sub, metadata !"round.downward", metadata !"fpexcept.strict") #2 + ; TODO: @llvm.experimental.constrained.fmuladd is not supported at the moment + ; %0 = tail call float @llvm.experimental.constrained.fmuladd.f32(float %mul, float %mul, float %mul, metadata !"round.tonearestaway", metadata !"fpexcept.strict") #2 + %r1 = tail call float @llvm.experimental.constrained.fma.f32(float %a, float %a, float %a, metadata !"round.dynamic", metadata !"fpexcept.strict") #2 + %r2 = tail call float @llvm.experimental.constrained.frem.f32(float %a, float %a, metadata !"round.dynamic", metadata !"fpexcept.strict") #2 + ret void +} + +; Function Attrs: inaccessiblememonly nounwind willreturn +declare float @llvm.experimental.constrained.fadd.f32(float, float, metadata, metadata) #1 + +; Function Attrs: inaccessiblememonly nounwind willreturn +declare float @llvm.experimental.constrained.fdiv.f32(float, float, metadata, metadata) #1 + +; Function Attrs: inaccessiblememonly nounwind willreturn +declare float @llvm.experimental.constrained.fsub.f32(float, float, metadata, metadata) #1 + +; Function Attrs: inaccessiblememonly nounwind willreturn +declare float @llvm.experimental.constrained.fmul.f32(float, float, metadata, metadata) #1 + +; Function Attrs: inaccessiblememonly nounwind willreturn +declare float @llvm.experimental.constrained.fmuladd.f32(float, float, float, metadata, metadata) #1 + +; Function Attrs: inaccessiblememonly nounwind willreturn +declare float @llvm.experimental.constrained.fma.f32(float, float, float, metadata, metadata) #1 + +; Function Attrs: inaccessiblememonly nounwind willreturn +declare float @llvm.experimental.constrained.frem.f32(float, float, metadata, metadata) #1 + +attributes #0 = { norecurse nounwind strictfp "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="test2.cl" "uniform-work-group-size"="true" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { inaccessiblememonly nounwind willreturn } +attributes #2 = { strictfp } + +!llvm.module.flags = !{!0} +!opencl.ocl.version = !{!1} +!opencl.spir.version = !{!2, !2} +!spirv.Source = !{!3} +!llvm.ident = !{!4} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{i32 1, i32 0} +!2 = !{i32 1, i32 2} +!3 = !{i32 4, i32 100000} +!4 = !{!"clang version 12.0.0 (https://github.com/c199914007/llvm.git f0c85a8adeb49638c01eee1451aa9b35462cbfd5)"} +!5 = !{i32 0, i32 0, i32 0} +!6 = !{!"none", !"none", !"none"} +!7 = !{!"float", !"int", !"uint"} +!8 = !{!"", !"", !""} +!9 = !{i32 -1, i32 -1, i32 -1} +!10 = !{float 2.500000e+00} From 34e83541bb4ca6036881adfef15295bfbe05bb8c Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Thu, 12 Dec 2024 14:26:27 -0800 Subject: [PATCH 2/6] generate OpDecorate from demangled postfix (_R*) of a builtin --- llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 18 +++++++++++------- .../SPIRV/instructions/integer-casts.ll | 10 ++++++++++ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 5ab3810900033..4377dd085522d 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -1298,14 +1298,18 @@ static void createRoundingModeDecoration(Instruction *I, IRBuilder<> &B) { LLVMContext &Ctx = I->getContext(); Type *Int32Ty = Type::getInt32Ty(Ctx); - SmallVector MDs = { - MDNode::get(Ctx, ConstantAsMetadata::get(ConstantInt::get( - Int32Ty, SPIRV::Decoration::FPRoundingMode))), - MDNode::get(Ctx, ConstantAsMetadata::get( - ConstantInt::get(Int32Ty, RoundingModeDeco)))}; setInsertPointAfterDef(B, I); - B.CreateIntrinsic(Intrinsic::spv_assign_decoration, {I->getType()}, - {I, MetadataAsValue::get(Ctx, MDNode::get(Ctx, MDs))}); + B.CreateIntrinsic( + Intrinsic::spv_assign_decoration, {I->getType()}, + {I, + MetadataAsValue::get( + Ctx, + MDNode::get( + Ctx, {MDNode::get( + Ctx, {ConstantAsMetadata::get(ConstantInt::get( + Int32Ty, SPIRV::Decoration::FPRoundingMode)), + ConstantAsMetadata::get(ConstantInt::get( + Int32Ty, RoundingModeDeco))})}))}); } Instruction *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) { diff --git a/llvm/test/CodeGen/SPIRV/instructions/integer-casts.ll b/llvm/test/CodeGen/SPIRV/instructions/integer-casts.ll index 6763c95b51bf5..7691d2d6b4dd1 100644 --- a/llvm/test/CodeGen/SPIRV/instructions/integer-casts.ll +++ b/llvm/test/CodeGen/SPIRV/instructions/integer-casts.ll @@ -24,12 +24,18 @@ ; CHECK-DAG: OpName [[ZEXT8_16v4:%.*]] "u8tou16v4" ; CHECK-DAG: OpName [[ZEXT16_32v4:%.*]] "u16tou32v4" +; CHECK-DAG: OpDecorate %[[#R17:]] FPRoundingMode RTZ +; CHECK-DAG: OpDecorate %[[#R18:]] FPRoundingMode RTE +; CHECK-DAG: OpDecorate %[[#R19:]] FPRoundingMode RTP +; CHECK-DAG: OpDecorate %[[#R20:]] FPRoundingMode RTN + ; CHECK-DAG: [[F32:%.*]] = OpTypeFloat 32 ; CHECK-DAG: [[F16:%.*]] = OpTypeFloat 16 ; CHECK-DAG: [[U64:%.*]] = OpTypeInt 64 0 ; CHECK-DAG: [[U32:%.*]] = OpTypeInt 32 0 ; CHECK-DAG: [[U16:%.*]] = OpTypeInt 16 0 ; CHECK-DAG: [[U8:%.*]] = OpTypeInt 8 0 +; CHECK-DAG: [[F32v2:%.*]] = OpTypeVector [[F32]] 2 ; CHECK-DAG: [[U32v4:%.*]] = OpTypeVector [[U32]] 4 ; CHECK-DAG: [[U16v4:%.*]] = OpTypeVector [[U16]] 4 ; CHECK-DAG: [[U8v4:%.*]] = OpTypeVector [[U8]] 4 @@ -254,6 +260,10 @@ define <4 x i32> @u16tou32v4(<4 x i16> %a) { ; CHECK: %[[#]] = OpSConvert [[U32v4]] %[[#]] ; CHECK: %[[#]] = OpConvertUToF [[F32]] %[[#]] ; CHECK: %[[#]] = OpConvertUToF [[F32]] %[[#]] +; CHECK: %[[#R17:]] = OpFConvert [[F32v2]] %[[#]] +; CHECK: %[[#R18:]] = OpFConvert [[F32v2]] %[[#]] +; CHECK: %[[#R19:]] = OpFConvert [[F32v2]] %[[#]] +; CHECK: %[[#R20:]] = OpFConvert [[F32v2]] %[[#]] ; CHECK: OpFunctionEnd define dso_local spir_kernel void @test_wrappers(ptr addrspace(4) %arg, i64 %arg_ptr, <4 x i8> %arg_v2) { %r1 = call spir_func i32 @__spirv_ConvertFToU(float 0.000000e+00) From 10ab51c5d66d345f0d399fec950c122f6b87bc27 Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Fri, 13 Dec 2024 03:40:54 -0800 Subject: [PATCH 3/6] implement arith Constrained Floating-Point Intrinsics via Tablegen; add float Saturation decoration --- .../SPIRV/MCTargetDesc/SPIRVMCCodeEmitter.cpp | 9 ++-- llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 41 +++++++++++++------ llvm/lib/Target/SPIRV/SPIRVInstrInfo.td | 6 +++ .../Target/SPIRV/SPIRVInstructionSelector.cpp | 12 ++---- llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp | 10 +++-- llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp | 7 ++-- llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 4 +- .../SPIRV/instructions/integer-casts.ll | 26 ++++++++---- .../llvm-intrinsics/constrained-arithmetic.ll | 15 +++---- 9 files changed, 78 insertions(+), 52 deletions(-) diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVMCCodeEmitter.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVMCCodeEmitter.cpp index 42567f695395e..68cc6a3a7aac1 100644 --- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVMCCodeEmitter.cpp +++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVMCCodeEmitter.cpp @@ -65,11 +65,10 @@ static bool hasType(const MCInst &MI, const MCInstrInfo &MII) { // If we define an output, and have at least one other argument. if (MCDesc.getNumDefs() == 1 && MCDesc.getNumOperands() >= 2) { // Check if we define an ID, and take a type as operand 1. - auto &DefOpInfo = MCDesc.operands()[0]; - auto &FirstArgOpInfo = MCDesc.operands()[1]; - return DefOpInfo.RegClass >= 0 && FirstArgOpInfo.RegClass >= 0 && - DefOpInfo.RegClass != SPIRV::TYPERegClassID && - FirstArgOpInfo.RegClass == SPIRV::TYPERegClassID; + return MCDesc.operands()[0].RegClass >= 0 && + MCDesc.operands()[1].RegClass >= 0 && + MCDesc.operands()[0].RegClass != SPIRV::TYPERegClassID && + MCDesc.operands()[1].RegClass == SPIRV::TYPERegClassID; } return false; } diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 4377dd085522d..433956f44917f 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -1293,23 +1293,35 @@ void SPIRVEmitIntrinsics::preprocessCompositeConstants(IRBuilder<> &B) { } } +static void createDecorationIntrinsic(Instruction *I, MDNode *Node, + IRBuilder<> &B) { + LLVMContext &Ctx = I->getContext(); + setInsertPointAfterDef(B, I); + B.CreateIntrinsic(Intrinsic::spv_assign_decoration, {I->getType()}, + {I, MetadataAsValue::get(Ctx, MDNode::get(Ctx, {Node}))}); +} + static void createRoundingModeDecoration(Instruction *I, unsigned RoundingModeDeco, IRBuilder<> &B) { LLVMContext &Ctx = I->getContext(); Type *Int32Ty = Type::getInt32Ty(Ctx); - setInsertPointAfterDef(B, I); - B.CreateIntrinsic( - Intrinsic::spv_assign_decoration, {I->getType()}, - {I, - MetadataAsValue::get( - Ctx, - MDNode::get( - Ctx, {MDNode::get( - Ctx, {ConstantAsMetadata::get(ConstantInt::get( - Int32Ty, SPIRV::Decoration::FPRoundingMode)), - ConstantAsMetadata::get(ConstantInt::get( - Int32Ty, RoundingModeDeco))})}))}); + MDNode *RoundingModeNode = MDNode::get( + Ctx, + {ConstantAsMetadata::get( + ConstantInt::get(Int32Ty, SPIRV::Decoration::FPRoundingMode)), + ConstantAsMetadata::get(ConstantInt::get(Int32Ty, RoundingModeDeco))}); + createDecorationIntrinsic(I, RoundingModeNode, B); +} + +static void createSaturatedConversionDecoration(Instruction *I, + IRBuilder<> &B) { + LLVMContext &Ctx = I->getContext(); + Type *Int32Ty = Type::getInt32Ty(Ctx); + MDNode *SaturatedConversionNode = + MDNode::get(Ctx, {ConstantAsMetadata::get(ConstantInt::get( + Int32Ty, SPIRV::Decoration::SaturatedConversion))}); + createDecorationIntrinsic(I, SaturatedConversionNode, B); } Instruction *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) { @@ -1912,10 +1924,13 @@ void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I, SmallVector Parts; S.split(Parts, "_", -1, false); if (Parts.size() > 1) { - // Convert the tip about rounding mode into a decoration record. + // Convert the info about rounding mode into a decoration record. unsigned RoundingModeDeco = roundingModeMDToDecorationConst(Parts[1]); if (RoundingModeDeco != std::numeric_limits::max()) createRoundingModeDecoration(CI, RoundingModeDeco, B); + // Check if the SaturatedConversion info is present. + if (Parts[1] == "sat") + createSaturatedConversionDecoration(CI, B); } } } diff --git a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td index d95803fea56a5..1bc35c6e57a4f 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td +++ b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td @@ -491,16 +491,20 @@ def OpFNegate: UnOpTyped<"OpFNegate", 127, fID, fneg>; def OpFNegateV: UnOpTyped<"OpFNegate", 127, vfID, fneg>; defm OpIAdd: BinOpTypedGen<"OpIAdd", 128, add, 0, 1>; defm OpFAdd: BinOpTypedGen<"OpFAdd", 129, fadd, 1, 1>; +defm OpStrictFAdd: BinOpTypedGen<"OpFAdd", 129, strict_fadd, 1, 1>; defm OpISub: BinOpTypedGen<"OpISub", 130, sub, 0, 1>; defm OpFSub: BinOpTypedGen<"OpFSub", 131, fsub, 1, 1>; +defm OpStrictFSub: BinOpTypedGen<"OpFSub", 131, strict_fsub, 1, 1>; defm OpIMul: BinOpTypedGen<"OpIMul", 132, mul, 0, 1>; defm OpFMul: BinOpTypedGen<"OpFMul", 133, fmul, 1, 1>; +defm OpStrictFMul: BinOpTypedGen<"OpFMul", 133, strict_fmul, 1, 1>; defm OpUDiv: BinOpTypedGen<"OpUDiv", 134, udiv, 0, 1>; defm OpSDiv: BinOpTypedGen<"OpSDiv", 135, sdiv, 0, 1>; defm OpFDiv: BinOpTypedGen<"OpFDiv", 136, fdiv, 1, 1>; +defm OpStrictFDiv: BinOpTypedGen<"OpFDiv", 136, strict_fdiv, 1, 1>; defm OpUMod: BinOpTypedGen<"OpUMod", 137, urem, 0, 1>; defm OpSRem: BinOpTypedGen<"OpSRem", 138, srem, 0, 1>; @@ -508,6 +512,8 @@ defm OpSRem: BinOpTypedGen<"OpSRem", 138, srem, 0, 1>; def OpSMod: BinOp<"OpSMod", 139>; defm OpFRem: BinOpTypedGen<"OpFRem", 140, frem, 1, 1>; +defm OpStrictFRem: BinOpTypedGen<"OpFRem", 140, strict_frem, 1, 1>; + def OpFMod: BinOp<"OpFMod", 141>; def OpVectorTimesScalar: BinOp<"OpVectorTimesScalar", 142>; diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 690b6b57211bb..42fb7038a20b7 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -557,19 +557,12 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg, case TargetOpcode::G_UCMP: return selectSUCmp(ResVReg, ResType, I, false); + case TargetOpcode::G_STRICT_FMA: case TargetOpcode::G_FMA: return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); - case TargetOpcode::G_STRICT_FSQRT: - case TargetOpcode::G_STRICT_FADD: - case TargetOpcode::G_STRICT_FSUB: - case TargetOpcode::G_STRICT_FMUL: - case TargetOpcode::G_STRICT_FDIV: - case TargetOpcode::G_STRICT_FREM: case TargetOpcode::G_STRICT_FLDEXP: - return false; - case TargetOpcode::G_STRICT_FMA: - return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); + return selectExtInst(ResVReg, ResType, I, CL::ldexp); case TargetOpcode::G_FPOW: return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); @@ -629,6 +622,7 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg, case TargetOpcode::G_FTANH: return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh); + case TargetOpcode::G_STRICT_FSQRT: case TargetOpcode::G_FSQRT: return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp index 583a95150ef0c..2950bc86072cf 100644 --- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp @@ -27,16 +27,21 @@ using namespace llvm::LegalityPredicates; static const std::set TypeFoldingSupportingOpcs = { TargetOpcode::G_ADD, TargetOpcode::G_FADD, + TargetOpcode::G_STRICT_FADD, TargetOpcode::G_SUB, TargetOpcode::G_FSUB, + TargetOpcode::G_STRICT_FSUB, TargetOpcode::G_MUL, TargetOpcode::G_FMUL, + TargetOpcode::G_STRICT_FMUL, TargetOpcode::G_SDIV, TargetOpcode::G_UDIV, TargetOpcode::G_FDIV, + TargetOpcode::G_STRICT_FDIV, TargetOpcode::G_SREM, TargetOpcode::G_UREM, TargetOpcode::G_FREM, + TargetOpcode::G_STRICT_FREM, TargetOpcode::G_FNEG, TargetOpcode::G_CONSTANT, TargetOpcode::G_FCONSTANT, @@ -219,10 +224,7 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) { .legalFor(allIntScalarsAndVectors) .legalIf(extendedScalarsAndVectors); - getActionDefinitionsBuilder(G_FMA).legalFor(allFloatScalarsAndVectors); - - getActionDefinitionsBuilder({G_STRICT_FSQRT, G_STRICT_FADD, G_STRICT_FSUB, G_STRICT_FMUL, - G_STRICT_FDIV, G_STRICT_FREM, G_STRICT_FMA}) + getActionDefinitionsBuilder({G_FMA, G_STRICT_FMA}) .legalFor(allFloatScalarsAndVectors); getActionDefinitionsBuilder(G_STRICT_FLDEXP) diff --git a/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp index 3373d8e24dab4..a78c490bc9faa 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp @@ -55,10 +55,11 @@ extern void processInstr(MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI, SPIRVGlobalRegistry *GR); } // namespace llvm -static bool isMetaInstrGET(unsigned Opcode) { +static bool isMetaInstr(unsigned Opcode) { return Opcode == SPIRV::GET_ID || Opcode == SPIRV::GET_fID || Opcode == SPIRV::GET_pID || Opcode == SPIRV::GET_vID || - Opcode == SPIRV::GET_vfID || Opcode == SPIRV::GET_vpID; + Opcode == SPIRV::GET_vfID || Opcode == SPIRV::GET_vpID || + Opcode == SPIRV::ASSIGN_TYPE; } static bool mayBeInserted(unsigned Opcode) { @@ -128,7 +129,7 @@ static void processNewInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR, if (isTypeFoldingSupported(Opcode)) { // Check if the instruction newly generated or already processed MachineInstr *NextMI = I.getNextNode(); - if (NextMI && isMetaInstrGET(NextMI->getOpcode())) + if (NextMI && isMetaInstr(NextMI->getOpcode())) continue; // Restore usual instructions pattern for the newly inserted // instruction diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp index ceccf55d1de4d..cc48115240490 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp @@ -458,8 +458,10 @@ void processInstr(MachineInstr &MI, MachineIRBuilder &MIB, assert(MI.getNumDefs() > 0 && MRI.hasOneUse(MI.getOperand(0).getReg())); MachineInstr &AssignTypeInst = *(MRI.use_instr_begin(MI.getOperand(0).getReg())); + SPIRVType *SpvTypeRes = GR->getSPIRVTypeForVReg(MI.getOperand(0).getReg()); auto NewReg = - createNewIdReg(nullptr, MI.getOperand(0).getReg(), MRI, *GR).first; + createNewIdReg(SpvTypeRes, MI.getOperand(0).getReg(), MRI, *GR).first; + GR->assignSPIRVTypeToVReg(SpvTypeRes, NewReg, MIB.getMF()); AssignTypeInst.getOperand(1).setReg(NewReg); MI.getOperand(0).setReg(NewReg); MIB.setInsertPt(*MI.getParent(), MI.getIterator()); diff --git a/llvm/test/CodeGen/SPIRV/instructions/integer-casts.ll b/llvm/test/CodeGen/SPIRV/instructions/integer-casts.ll index 7691d2d6b4dd1..6a4b4f593bf3b 100644 --- a/llvm/test/CodeGen/SPIRV/instructions/integer-casts.ll +++ b/llvm/test/CodeGen/SPIRV/instructions/integer-casts.ll @@ -14,6 +14,12 @@ ; CHECK-DAG: OpName [[ZEXT8_16:%.*]] "u8tou16" ; CHECK-DAG: OpName [[ZEXT16_32:%.*]] "u16tou32" +; CHECK-DAG: OpName %[[#R17:]] "r17" +; CHECK-DAG: OpName %[[#R18:]] "r18" +; CHECK-DAG: OpName %[[#R19:]] "r19" +; CHECK-DAG: OpName %[[#R20:]] "r20" +; CHECK-DAG: OpName %[[#R21:]] "r21" + ; CHECK-DAG: OpName [[TRUNC32_16v4:%.*]] "i32toi16v4" ; CHECK-DAG: OpName [[TRUNC32_8v4:%.*]] "i32toi8v4" ; CHECK-DAG: OpName [[TRUNC16_8v4:%.*]] "i16toi8v4" @@ -24,10 +30,11 @@ ; CHECK-DAG: OpName [[ZEXT8_16v4:%.*]] "u8tou16v4" ; CHECK-DAG: OpName [[ZEXT16_32v4:%.*]] "u16tou32v4" -; CHECK-DAG: OpDecorate %[[#R17:]] FPRoundingMode RTZ -; CHECK-DAG: OpDecorate %[[#R18:]] FPRoundingMode RTE -; CHECK-DAG: OpDecorate %[[#R19:]] FPRoundingMode RTP -; CHECK-DAG: OpDecorate %[[#R20:]] FPRoundingMode RTN +; CHECK-DAG: OpDecorate %[[#R17]] FPRoundingMode RTZ +; CHECK-DAG: OpDecorate %[[#R18]] FPRoundingMode RTE +; CHECK-DAG: OpDecorate %[[#R19]] FPRoundingMode RTP +; CHECK-DAG: OpDecorate %[[#R20]] FPRoundingMode RTN +; CHECK-DAG: OpDecorate %[[#R21]] SaturatedConversion ; CHECK-DAG: [[F32:%.*]] = OpTypeFloat 32 ; CHECK-DAG: [[F16:%.*]] = OpTypeFloat 16 @@ -260,10 +267,11 @@ define <4 x i32> @u16tou32v4(<4 x i16> %a) { ; CHECK: %[[#]] = OpSConvert [[U32v4]] %[[#]] ; CHECK: %[[#]] = OpConvertUToF [[F32]] %[[#]] ; CHECK: %[[#]] = OpConvertUToF [[F32]] %[[#]] -; CHECK: %[[#R17:]] = OpFConvert [[F32v2]] %[[#]] -; CHECK: %[[#R18:]] = OpFConvert [[F32v2]] %[[#]] -; CHECK: %[[#R19:]] = OpFConvert [[F32v2]] %[[#]] -; CHECK: %[[#R20:]] = OpFConvert [[F32v2]] %[[#]] +; CHECK: %[[#R17]] = OpFConvert [[F32v2]] %[[#]] +; CHECK: %[[#R18]] = OpFConvert [[F32v2]] %[[#]] +; CHECK: %[[#R19]] = OpFConvert [[F32v2]] %[[#]] +; CHECK: %[[#R20]] = OpFConvert [[F32v2]] %[[#]] +; CHECK: %[[#R21]] = OpConvertFToU [[U8]] %[[#]] ; CHECK: OpFunctionEnd define dso_local spir_kernel void @test_wrappers(ptr addrspace(4) %arg, i64 %arg_ptr, <4 x i8> %arg_v2) { %r1 = call spir_func i32 @__spirv_ConvertFToU(float 0.000000e+00) @@ -286,6 +294,7 @@ define dso_local spir_kernel void @test_wrappers(ptr addrspace(4) %arg, i64 %arg %r18 = call spir_func <2 x float> @_Z28__spirv_FConvert_Rfloat2_rteDv2_DF16_(<2 x half> noundef ) %r19 = call spir_func <2 x float> @_Z28__spirv_FConvert_Rfloat2_rtpDv2_DF16_(<2 x half> noundef ) %r20 = call spir_func <2 x float> @_Z28__spirv_FConvert_Rfloat2_rtnDv2_DF16_(<2 x half> noundef ) + %r21 = call spir_func i8 @_Z30__spirv_ConvertFToU_Ruchar_satf(float noundef 42.0) ret void } @@ -309,3 +318,4 @@ declare dso_local spir_func <2 x float> @_Z28__spirv_FConvert_Rfloat2_rtzDv2_DF1 declare dso_local spir_func <2 x float> @_Z28__spirv_FConvert_Rfloat2_rteDv2_DF16_(<2 x half> noundef) declare dso_local spir_func <2 x float> @_Z28__spirv_FConvert_Rfloat2_rtpDv2_DF16_(<2 x half> noundef) declare dso_local spir_func <2 x float> @_Z28__spirv_FConvert_Rfloat2_rtnDv2_DF16_(<2 x half> noundef) +declare dso_local spir_func i8 @_Z30__spirv_ConvertFToU_Ruchar_satf(float) diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/constrained-arithmetic.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/constrained-arithmetic.ll index e30a2fef1d8cd..8b8d9973943b8 100644 --- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/constrained-arithmetic.ll +++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/constrained-arithmetic.ll @@ -21,8 +21,6 @@ ; CHECK: OpFDiv %[[#]] %[[#di]] ; CHECK: OpFSub %[[#]] %[[#su]] ; CHECK: OpFMul %[[#]] %[[#mu]] -; CHECK: OpFMul -; CHECK: OpFAdd ; CHECK: OpExtInst %[[#]] %[[#]] %[[#]] fma ; CHECK: OpFRem @@ -30,13 +28,12 @@ define dso_local spir_kernel void @test(float %a, i32 %in, i32 %ui) local_unnamed_addr #0 !kernel_arg_addr_space !5 !kernel_arg_access_qual !6 !kernel_arg_type !7 !kernel_arg_base_type !7 !kernel_arg_type_qual !8 !kernel_arg_buffer_location !9 { entry: %add = tail call float @llvm.experimental.constrained.fadd.f32(float %a, float %a, metadata !"round.tonearest", metadata !"fpexcept.strict") #2 - %div = tail call float @llvm.experimental.constrained.fdiv.f32(float %add, float %add, metadata !"round.towardzero", metadata !"fpexcept.strict") #2, !fpmath !10 - %sub = tail call float @llvm.experimental.constrained.fsub.f32(float %div, float %div, metadata !"round.upward", metadata !"fpexcept.strict") #2 - %mul = tail call float @llvm.experimental.constrained.fmul.f32(float %sub, float %sub, metadata !"round.downward", metadata !"fpexcept.strict") #2 - ; TODO: @llvm.experimental.constrained.fmuladd is not supported at the moment - ; %0 = tail call float @llvm.experimental.constrained.fmuladd.f32(float %mul, float %mul, float %mul, metadata !"round.tonearestaway", metadata !"fpexcept.strict") #2 - %r1 = tail call float @llvm.experimental.constrained.fma.f32(float %a, float %a, float %a, metadata !"round.dynamic", metadata !"fpexcept.strict") #2 - %r2 = tail call float @llvm.experimental.constrained.frem.f32(float %a, float %a, metadata !"round.dynamic", metadata !"fpexcept.strict") #2 + %add2 = fadd float %a, %a +; %div = tail call float @llvm.experimental.constrained.fdiv.f32(float %a, float %a, metadata !"round.towardzero", metadata !"fpexcept.strict") #2, !fpmath !10 +; %sub = tail call float @llvm.experimental.constrained.fsub.f32(float %a, float %a, metadata !"round.upward", metadata !"fpexcept.strict") #2 +; %mul = tail call float @llvm.experimental.constrained.fmul.f32(float %a, float %a, metadata !"round.downward", metadata !"fpexcept.strict") #2 +; %r1 = tail call float @llvm.experimental.constrained.fma.f32(float %a, float %a, float %a, metadata !"round.dynamic", metadata !"fpexcept.strict") #2 +; %r2 = tail call float @llvm.experimental.constrained.frem.f32(float %a, float %a, metadata !"round.dynamic", metadata !"fpexcept.strict") #2 ret void } From 8cf4bd91d49346c5eddcd0f74683df27576a3e7a Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Fri, 13 Dec 2024 08:30:45 -0800 Subject: [PATCH 4/6] improve instruction selection --- .../Target/SPIRV/SPIRVInstructionSelector.cpp | 33 +++++- llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp | 3 +- .../llvm-intrinsics/constrained-arithmetic.ll | 105 ++++++------------ 3 files changed, 67 insertions(+), 74 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 42fb7038a20b7..856caf2074fba 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -61,6 +61,7 @@ class SPIRVInstructionSelector : public InstructionSelector { /// We need to keep track of the number we give to anonymous global values to /// generate the same name every time when this is needed. mutable DenseMap UnnamedGlobalIDs; + SmallPtrSet DeadMIs; public: SPIRVInstructionSelector(const SPIRVTargetMachine &TM, @@ -382,6 +383,24 @@ static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); // Defined in SPIRVLegalizerInfo.cpp. extern bool isTypeFoldingSupported(unsigned Opcode); +bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI) { + for (const auto &MO : MI.all_defs()) { + Register Reg = MO.getReg(); + if (Reg.isPhysical() || !MRI.use_nodbg_empty(Reg)) + return false; + } + if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE || MI.isFakeUse() || + MI.isLifetimeMarker()) + return false; + if (MI.isPHI()) + return true; + if (MI.mayStore() || MI.isCall() || + (MI.mayLoad() && MI.hasOrderedMemoryRef()) || MI.isPosition() || + MI.isDebugInstr() || MI.isTerminator() || MI.isJumpTableDebugInfo()) + return false; + return true; +} + bool SPIRVInstructionSelector::select(MachineInstr &I) { resetVRegsType(*I.getParent()->getParent()); @@ -404,8 +423,11 @@ bool SPIRVInstructionSelector::select(MachineInstr &I) { } }); assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); - if (Res) + if (Res) { + if (!isTriviallyDead(*Def, *MRI) && isDead(*Def, *MRI)) + DeadMIs.insert(Def); return Res; + } } MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg)); MRI->replaceRegWith(SrcReg, DstReg); @@ -418,6 +440,15 @@ bool SPIRVInstructionSelector::select(MachineInstr &I) { return constrainSelectedInstRegOperands(I, TII, TRI, RBI); } + if (DeadMIs.contains(&I)) { + // if the instruction has been already made dead by folding it away + // erase it + LLVM_DEBUG(dbgs() << "Instruction is folded and dead.\n"); + salvageDebugInfo(*MRI, I); + I.eraseFromParent(); + return true; + } + if (I.getNumOperands() != I.getNumExplicitOperands()) { LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); return false; diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp index 2950bc86072cf..80464e14b2cc2 100644 --- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp @@ -228,8 +228,7 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) { .legalFor(allFloatScalarsAndVectors); getActionDefinitionsBuilder(G_STRICT_FLDEXP) - .legalForCartesianProduct(allFloatScalarsAndVectors, - allFloatScalarsAndVectors); + .legalForCartesianProduct(allFloatScalarsAndVectors, allIntScalars); getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI}) .legalForCartesianProduct(allIntScalarsAndVectors, diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/constrained-arithmetic.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/constrained-arithmetic.ll index 8b8d9973943b8..11bedfa605f9b 100644 --- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/constrained-arithmetic.ll +++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/constrained-arithmetic.ll @@ -1,81 +1,44 @@ -; Adapted from https://github.com/KhronosGroup/SPIRV-LLVM-Translator/tree/main/test/llvm-intrinsics - ; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} -; CHECK: OpName %[[#ad:]] "add" -; CHECK: OpName %[[#di:]] "div" -; CHECK: OpName %[[#su:]] "sub" -; CHECK: OpName %[[#mu:]] "mul" - -; CHECK-NOT: OpDecorate %[[#]] FPRoundingMode - -; CHECK-DAG: OpDecorate %[[#ad]] FPRoundingMode 0 -; CHECK-DAG: OpDecorate %[[#di]] FPRoundingMode 1 -; CHECK-DAG: OpDecorate %[[#su]] FPRoundingMode 2 -; CHECK-DAG: OpDecorate %[[#mu]] FPRoundingMode 3 - -; CHECK-NOT: OpDecorate %[[#]] FPRoundingMode - -; CHECK: OpFAdd %[[#]] %[[#ad]] -; CHECK: OpFDiv %[[#]] %[[#di]] -; CHECK: OpFSub %[[#]] %[[#su]] -; CHECK: OpFMul %[[#]] %[[#mu]] -; CHECK: OpExtInst %[[#]] %[[#]] %[[#]] fma +; CHECK-DAG: OpName %[[#r1:]] "r1" +; CHECK-DAG: OpName %[[#r2:]] "r2" +; CHECK-DAG: OpName %[[#r3:]] "r3" +; CHECK-DAG: OpName %[[#r4:]] "r4" +; CHECK-DAG: OpName %[[#r5:]] "r5" +; CHECK-DAG: OpName %[[#r6:]] "r6" + +; CHECK-NOT: OpDecorate %[[#r5]] FPRoundingMode +; CHECK-NOT: OpDecorate %[[#r6]] FPRoundingMode + +; CHECK-DAG: OpDecorate %[[#r1]] FPRoundingMode RTE +; CHECK-DAG: OpDecorate %[[#r2]] FPRoundingMode RTZ +; CHECK-DAG: OpDecorate %[[#r4]] FPRoundingMode RTN +; CHECK-DAG: OpDecorate %[[#r3]] FPRoundingMode RTP + +; CHECK: OpFAdd %[[#]] %[[#]] +; CHECK: OpFDiv %[[#]] %[[#]] +; CHECK: OpFSub %[[#]] %[[#]] +; CHECK: OpFMul %[[#]] %[[#]] +; CHECK: OpExtInst %[[#]] %[[#]] fma %[[#]] %[[#]] %[[#]] ; CHECK: OpFRem ; Function Attrs: norecurse nounwind strictfp -define dso_local spir_kernel void @test(float %a, i32 %in, i32 %ui) local_unnamed_addr #0 !kernel_arg_addr_space !5 !kernel_arg_access_qual !6 !kernel_arg_type !7 !kernel_arg_base_type !7 !kernel_arg_type_qual !8 !kernel_arg_buffer_location !9 { +define dso_local spir_kernel void @test(float %a, i32 %in, i32 %ui) { entry: - %add = tail call float @llvm.experimental.constrained.fadd.f32(float %a, float %a, metadata !"round.tonearest", metadata !"fpexcept.strict") #2 - %add2 = fadd float %a, %a -; %div = tail call float @llvm.experimental.constrained.fdiv.f32(float %a, float %a, metadata !"round.towardzero", metadata !"fpexcept.strict") #2, !fpmath !10 -; %sub = tail call float @llvm.experimental.constrained.fsub.f32(float %a, float %a, metadata !"round.upward", metadata !"fpexcept.strict") #2 -; %mul = tail call float @llvm.experimental.constrained.fmul.f32(float %a, float %a, metadata !"round.downward", metadata !"fpexcept.strict") #2 -; %r1 = tail call float @llvm.experimental.constrained.fma.f32(float %a, float %a, float %a, metadata !"round.dynamic", metadata !"fpexcept.strict") #2 -; %r2 = tail call float @llvm.experimental.constrained.frem.f32(float %a, float %a, metadata !"round.dynamic", metadata !"fpexcept.strict") #2 + %r1 = tail call float @llvm.experimental.constrained.fadd.f32(float %a, float %a, metadata !"round.tonearest", metadata !"fpexcept.strict") + %r2 = tail call float @llvm.experimental.constrained.fdiv.f32(float %a, float %a, metadata !"round.towardzero", metadata !"fpexcept.strict") + %r3 = tail call float @llvm.experimental.constrained.fsub.f32(float %a, float %a, metadata !"round.upward", metadata !"fpexcept.strict") + %r4 = tail call float @llvm.experimental.constrained.fmul.f32(float %a, float %a, metadata !"round.downward", metadata !"fpexcept.strict") + %r5 = tail call float @llvm.experimental.constrained.fma.f32(float %a, float %a, float %a, metadata !"round.dynamic", metadata !"fpexcept.strict") + %r6 = tail call float @llvm.experimental.constrained.frem.f32(float %a, float %a, metadata !"round.dynamic", metadata !"fpexcept.strict") ret void } -; Function Attrs: inaccessiblememonly nounwind willreturn -declare float @llvm.experimental.constrained.fadd.f32(float, float, metadata, metadata) #1 - -; Function Attrs: inaccessiblememonly nounwind willreturn -declare float @llvm.experimental.constrained.fdiv.f32(float, float, metadata, metadata) #1 - -; Function Attrs: inaccessiblememonly nounwind willreturn -declare float @llvm.experimental.constrained.fsub.f32(float, float, metadata, metadata) #1 - -; Function Attrs: inaccessiblememonly nounwind willreturn -declare float @llvm.experimental.constrained.fmul.f32(float, float, metadata, metadata) #1 - -; Function Attrs: inaccessiblememonly nounwind willreturn -declare float @llvm.experimental.constrained.fmuladd.f32(float, float, float, metadata, metadata) #1 - -; Function Attrs: inaccessiblememonly nounwind willreturn -declare float @llvm.experimental.constrained.fma.f32(float, float, float, metadata, metadata) #1 - -; Function Attrs: inaccessiblememonly nounwind willreturn -declare float @llvm.experimental.constrained.frem.f32(float, float, metadata, metadata) #1 - -attributes #0 = { norecurse nounwind strictfp "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="test2.cl" "uniform-work-group-size"="true" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #1 = { inaccessiblememonly nounwind willreturn } -attributes #2 = { strictfp } - -!llvm.module.flags = !{!0} -!opencl.ocl.version = !{!1} -!opencl.spir.version = !{!2, !2} -!spirv.Source = !{!3} -!llvm.ident = !{!4} - -!0 = !{i32 1, !"wchar_size", i32 4} -!1 = !{i32 1, i32 0} -!2 = !{i32 1, i32 2} -!3 = !{i32 4, i32 100000} -!4 = !{!"clang version 12.0.0 (https://github.com/c199914007/llvm.git f0c85a8adeb49638c01eee1451aa9b35462cbfd5)"} -!5 = !{i32 0, i32 0, i32 0} -!6 = !{!"none", !"none", !"none"} -!7 = !{!"float", !"int", !"uint"} -!8 = !{!"", !"", !""} -!9 = !{i32 -1, i32 -1, i32 -1} -!10 = !{float 2.500000e+00} +declare float @llvm.experimental.constrained.fadd.f32(float, float, metadata, metadata) +declare float @llvm.experimental.constrained.fdiv.f32(float, float, metadata, metadata) +declare float @llvm.experimental.constrained.fsub.f32(float, float, metadata, metadata) +declare float @llvm.experimental.constrained.fmul.f32(float, float, metadata, metadata) +declare float @llvm.experimental.constrained.fmuladd.f32(float, float, float, metadata, metadata) +declare float @llvm.experimental.constrained.fma.f32(float, float, float, metadata, metadata) +declare float @llvm.experimental.constrained.frem.f32(float, float, metadata, metadata) From 9e0a5a1c2497f92a76cc1c994e06464ca5376a8b Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Fri, 13 Dec 2024 12:31:14 -0800 Subject: [PATCH 5/6] add Decoration for SPV_INTEL_float_controls2; fix postlegalizer --- llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 14 ++++---- llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp | 2 ++ llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp | 32 ++++++------------- .../lib/Target/SPIRV/SPIRVSymbolicOperands.td | 3 ++ 4 files changed, 21 insertions(+), 30 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index 5f72a41ddb864..3e913646d57c8 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -126,14 +126,14 @@ SPIRVType *SPIRVGlobalRegistry::getOpTypeInt(unsigned Width, Width = adjustOpTypeIntWidth(Width); const SPIRVSubtarget &ST = cast(MIRBuilder.getMF().getSubtarget()); - if (ST.canUseExtension( - SPIRV::Extension::SPV_INTEL_arbitrary_precision_integers)) { - MIRBuilder.buildInstr(SPIRV::OpExtension) - .addImm(SPIRV::Extension::SPV_INTEL_arbitrary_precision_integers); - MIRBuilder.buildInstr(SPIRV::OpCapability) - .addImm(SPIRV::Capability::ArbitraryPrecisionIntegersINTEL); - } return createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) { + if (ST.canUseExtension( + SPIRV::Extension::SPV_INTEL_arbitrary_precision_integers)) { + MIRBuilder.buildInstr(SPIRV::OpExtension) + .addImm(SPIRV::Extension::SPV_INTEL_arbitrary_precision_integers); + MIRBuilder.buildInstr(SPIRV::OpCapability) + .addImm(SPIRV::Capability::ArbitraryPrecisionIntegersINTEL); + } return MIRBuilder.buildInstr(SPIRV::OpTypeInt) .addDef(createTypeVReg(MIRBuilder)) .addImm(Width) diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp index 80464e14b2cc2..b22027cd2cb93 100644 --- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp @@ -24,6 +24,7 @@ using namespace llvm; using namespace llvm::LegalizeActions; using namespace llvm::LegalityPredicates; +// clang-format off static const std::set TypeFoldingSupportingOpcs = { TargetOpcode::G_ADD, TargetOpcode::G_FADD, @@ -54,6 +55,7 @@ static const std::set TypeFoldingSupportingOpcs = { TargetOpcode::G_SELECT, TargetOpcode::G_EXTRACT_VECTOR_ELT, }; +// clang-format on bool isTypeFoldingSupported(unsigned Opcode) { return TypeFoldingSupportingOpcs.count(Opcode) > 0; diff --git a/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp index a78c490bc9faa..6027d24636d69 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp @@ -55,13 +55,6 @@ extern void processInstr(MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI, SPIRVGlobalRegistry *GR); } // namespace llvm -static bool isMetaInstr(unsigned Opcode) { - return Opcode == SPIRV::GET_ID || Opcode == SPIRV::GET_fID || - Opcode == SPIRV::GET_pID || Opcode == SPIRV::GET_vID || - Opcode == SPIRV::GET_vfID || Opcode == SPIRV::GET_vpID || - Opcode == SPIRV::ASSIGN_TYPE; -} - static bool mayBeInserted(unsigned Opcode) { switch (Opcode) { case TargetOpcode::G_SMAX: @@ -112,27 +105,20 @@ static void processNewInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR, // registers, we must decorate them as if they were introduced in a // non-automatic way Register ResVReg = I.getOperand(0).getReg(); - SPIRVType *ResVType = GR->getSPIRVTypeForVReg(ResVReg); // Check if the register defined by the instruction is newly generated // or already processed - if (!ResVType) { - // Set type of the defined register - ResVType = GR->getSPIRVTypeForVReg(I.getOperand(1).getReg()); - // Check if we have type defined for operands of the new instruction - if (!ResVType) - continue; - // Set type & class - setRegClassType(ResVReg, ResVType, GR, &MRI, *GR->CurMF, true); - } + if (MRI.getRegClassOrNull(ResVReg)) + continue; + assert(GR->getSPIRVTypeForVReg(ResVReg) == nullptr); + // Check if we have type defined for operands of the new instruction + SPIRVType *ResVType = GR->getSPIRVTypeForVReg(I.getOperand(1).getReg()); + if (!ResVType) + continue; + // Set type & class + setRegClassType(ResVReg, ResVType, GR, &MRI, *GR->CurMF, true); // If this is a simple operation that is to be reduced by TableGen // definition we must apply some of pre-legalizer rules here if (isTypeFoldingSupported(Opcode)) { - // Check if the instruction newly generated or already processed - MachineInstr *NextMI = I.getNextNode(); - if (NextMI && isMetaInstr(NextMI->getOpcode())) - continue; - // Restore usual instructions pattern for the newly inserted - // instruction insertAssignInstr(ResVReg, nullptr, ResVType, GR, MIB, MRI); processInstr(I, MIB, MRI, GR); } diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td index 3071df0e28c56..fec3cb0091bf5 100644 --- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td +++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td @@ -1250,6 +1250,9 @@ defm CacheControlStoreINTEL : DecorationOperand<6443, 0, 0, [], [CacheControlsIN defm HostAccessINTEL : DecorationOperand<6188, 0, 0, [], [GlobalVariableHostAccessINTEL]>; defm InitModeINTEL : DecorationOperand<6190, 0, 0, [], [GlobalVariableFPGADecorationsINTEL]>; defm ImplementInRegisterMapINTEL : DecorationOperand<6191, 0, 0, [], [GlobalVariableFPGADecorationsINTEL]>; +defm FunctionRoundingModeINTEL : DecorationOperand<5822, 0, 0, [], [FunctionFloatControlINTEL]>; +defm FunctionDenormModeINTEL : DecorationOperand<5823, 0, 0, [], [FunctionFloatControlINTEL]>; +defm FunctionFloatingPointModeINTEL : DecorationOperand<6080, 0, 0, [], [FunctionFloatControlINTEL]>; //===----------------------------------------------------------------------===// // Multiclass used to define BuiltIn enum values and at the same time From cd9768b8e46d0f365fe8940b794b7f35181a6f15 Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Fri, 13 Dec 2024 15:24:03 -0800 Subject: [PATCH 6/6] simplify pre- and post-legalizer logics; fix wrong v-reg class in grp builtins --- llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp | 13 +++++-- llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 38 ++------------------- 2 files changed, 14 insertions(+), 37 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp index 806eadffca6e7..4bfa51e2cccdd 100644 --- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp @@ -587,6 +587,15 @@ static Register buildScopeReg(Register CLScopeRegister, return buildConstantIntReg32(Scope, MIRBuilder, GR); } +static void setRegClassIfNull(Register Reg, MachineRegisterInfo *MRI, + SPIRVGlobalRegistry *GR) { + if (MRI->getRegClassOrNull(Reg)) + return; + SPIRVType *SpvType = GR->getSPIRVTypeForVReg(Reg); + MRI->setRegClass(Reg, + SpvType ? GR->getRegClass(SpvType) : &SPIRV::iIDRegClass); +} + static Register buildMemSemanticsReg(Register SemanticsRegister, Register PtrRegister, unsigned &Semantics, MachineIRBuilder &MIRBuilder, @@ -1164,7 +1173,7 @@ static bool generateGroupInst(const SPIRV::IncomingCall *Call, MIRBuilder.buildInstr(TargetOpcode::G_BUILD_VECTOR).addDef(VecReg); for (unsigned i = 1; i < Call->Arguments.size(); i++) { MIB.addUse(Call->Arguments[i]); - MRI->setRegClass(Call->Arguments[i], &SPIRV::iIDRegClass); + setRegClassIfNull(Call->Arguments[i], MRI, GR); } insertAssignInstr(VecReg, nullptr, VecType, GR, MIRBuilder, MIRBuilder.getMF().getRegInfo()); @@ -1180,7 +1189,7 @@ static bool generateGroupInst(const SPIRV::IncomingCall *Call, MIB.addImm(GroupBuiltin->GroupOperation); if (Call->Arguments.size() > 0) { MIB.addUse(Arg0.isValid() ? Arg0 : Call->Arguments[0]); - MRI->setRegClass(Call->Arguments[0], &SPIRV::iIDRegClass); + setRegClassIfNull(Call->Arguments[0], MRI, GR); if (VecReg.isValid()) MIB.addUse(VecReg); else diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp index cc48115240490..8357c30d6949c 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp @@ -287,6 +287,7 @@ static SPIRVType *propagateSPIRVType(MachineInstr *MI, SPIRVGlobalRegistry *GR, SpvType = GR->getSPIRVTypeForVReg(Reg); if (!SpvType) { switch (MI->getOpcode()) { + case TargetOpcode::G_FCONSTANT: case TargetOpcode::G_CONSTANT: { MIB.setInsertPt(*MI->getParent(), MI); Type *Ty = MI->getOperand(1).getCImm()->getType(); @@ -455,15 +456,6 @@ Register insertAssignInstr(Register Reg, Type *Ty, SPIRVType *SpvType, void processInstr(MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI, SPIRVGlobalRegistry *GR) { - assert(MI.getNumDefs() > 0 && MRI.hasOneUse(MI.getOperand(0).getReg())); - MachineInstr &AssignTypeInst = - *(MRI.use_instr_begin(MI.getOperand(0).getReg())); - SPIRVType *SpvTypeRes = GR->getSPIRVTypeForVReg(MI.getOperand(0).getReg()); - auto NewReg = - createNewIdReg(SpvTypeRes, MI.getOperand(0).getReg(), MRI, *GR).first; - GR->assignSPIRVTypeToVReg(SpvTypeRes, NewReg, MIB.getMF()); - AssignTypeInst.getOperand(1).setReg(NewReg); - MI.getOperand(0).setReg(NewReg); MIB.setInsertPt(*MI.getParent(), MI.getIterator()); for (auto &Op : MI.operands()) { if (!Op.isReg() || Op.isDef()) @@ -676,34 +668,10 @@ static void processInstrsWithTypeFolding(MachineFunction &MF, SPIRVGlobalRegistry *GR, MachineIRBuilder MIB) { MachineRegisterInfo &MRI = MF.getRegInfo(); - for (MachineBasicBlock &MBB : MF) { - for (MachineInstr &MI : MBB) { + for (MachineBasicBlock &MBB : MF) + for (MachineInstr &MI : MBB) if (isTypeFoldingSupported(MI.getOpcode())) processInstr(MI, MIB, MRI, GR); - } - } - - for (MachineBasicBlock &MBB : MF) { - for (MachineInstr &MI : MBB) { - // We need to rewrite dst types for ASSIGN_TYPE instrs to be able - // to perform tblgen'erated selection and we can't do that on Legalizer - // as it operates on gMIR only. - if (MI.getOpcode() != SPIRV::ASSIGN_TYPE) - continue; - Register SrcReg = MI.getOperand(1).getReg(); - unsigned Opcode = MRI.getVRegDef(SrcReg)->getOpcode(); - if (!isTypeFoldingSupported(Opcode)) - continue; - Register DstReg = MI.getOperand(0).getReg(); - // Don't need to reset type of register holding constant and used in - // G_ADDRSPACE_CAST, since it breaks legalizer. - if (Opcode == TargetOpcode::G_CONSTANT && MRI.hasOneUse(DstReg)) { - MachineInstr &UseMI = *MRI.use_instr_begin(DstReg); - if (UseMI.getOpcode() == TargetOpcode::G_ADDRSPACE_CAST) - continue; - } - } - } } static Register