Merge pull request #230 from haonanya/haonanya/branch-80/lower-llvm.fmuladd

Add possibility to lower llvm.fmuladd into mad from OpenCL extinst

Author: fzou1

patches/spirv/0002-Add-possibility-to-lower-llvm.fmuladd.patch

@@ -0,0 +1,170 @@
+From 18038368f25cd2b06a55083a8b7030134bd75350 Mon Sep 17 00:00:00 2001
+From: Alexey Sachkov <[email protected]>
+Date: Thu, 17 Dec 2020 11:47:07 +0300
+Subject: [PATCH] Add possibility to lower llvm.fmuladd into mad from OpenCL
+ extinst (#858)
+
+---
+ include/LLVMSPIRVOpts.h          | 12 ++++++++++++
+ lib/SPIRV/SPIRVWriter.cpp        | 13 +++++++++++++
+ lib/SPIRV/libSPIRV/SPIRVModule.h |  4 ++++
+ test/llvm.fmuladd.ll             | 33 ++++++++++++++++++++------------
+ tools/llvm-spirv/llvm-spirv.cpp  | 16 ++++++++++++++++
+ 5 files changed, 66 insertions(+), 12 deletions(-)
+
+diff --git a/include/LLVMSPIRVOpts.h b/include/LLVMSPIRVOpts.h
+index 3e36375..4d7d5f0 100644
+--- a/include/LLVMSPIRVOpts.h
++++ b/include/LLVMSPIRVOpts.h
+@@ -107,6 +107,14 @@ public:
+ 
+   void setDebugInfoEIS(DebugInfoEIS EIS) { DebugInfoVersion = EIS; }
+ 
++  bool shouldReplaceLLVMFmulAddWithOpenCLMad() const noexcept {
++    return ReplaceLLVMFmulAddWithOpenCLMad;
++  }
++
++  void setReplaceLLVMFmulAddWithOpenCLMad(bool Value) noexcept {
++    ReplaceLLVMFmulAddWithOpenCLMad = Value;
++  }
++
+ private:
+   // Common translation options
+   VersionNumber MaxVersion = VersionNumber::MaximumVersion;
+@@ -124,6 +132,10 @@ private:
+   bool SPIRVAllowUnknownIntrinsics = false;
+ 
+   DebugInfoEIS DebugInfoVersion = DebugInfoEIS::SPIRV_Debug;
++
++  // Controls whether llvm.fmuladd.* should be replaced with mad from OpenCL
++  // extended instruction set or with a simple fmul + fadd
++  bool ReplaceLLVMFmulAddWithOpenCLMad = true;
+ };
+ 
+ } // namespace SPIRV
+diff --git a/lib/SPIRV/SPIRVWriter.cpp b/lib/SPIRV/SPIRVWriter.cpp
+index 810ba1f..332dbdb 100644
+--- a/lib/SPIRV/SPIRVWriter.cpp
++++ b/lib/SPIRV/SPIRVWriter.cpp
+@@ -1260,6 +1260,19 @@ SPIRVValue *LLVMToSPIRV::transIntrinsicInst(IntrinsicInst *II,
+     // For llvm.fmuladd.* fusion is not guaranteed. If a fused multiply-add
+     // is required the corresponding llvm.fma.* intrinsic function should be
+     // used instead.
++    // If allowed, let's replace llvm.fmuladd.* with mad from OpenCL extended
++    // instruction set, as it has the same semantic for FULL_PROFILE OpenCL
++    // devices (implementation-defined for EMBEDDED_PROFILE).
++    if (BM->shouldReplaceLLVMFmulAddWithOpenCLMad()) {
++      std::vector<SPIRVValue *> Ops{transValue(II->getArgOperand(0), BB),
++                                    transValue(II->getArgOperand(1), BB),
++                                    transValue(II->getArgOperand(2), BB)};
++      return BM->addExtInst(transType(II->getType()),
++                            BM->getExtInstSetId(SPIRVEIS_OpenCL),
++                            OpenCLLIB::Mad, Ops, BB);
++    }
++
++    // Otherwise, just break llvm.fmuladd.* into a pair of fmul + fadd
+     SPIRVType *Ty = transType(II->getType());
+     SPIRVValue *Mul =
+         BM->addBinaryInst(OpFMul, Ty, transValue(II->getArgOperand(0), BB),
+diff --git a/lib/SPIRV/libSPIRV/SPIRVModule.h b/lib/SPIRV/libSPIRV/SPIRVModule.h
+index 4460821..c3ea082 100644
+--- a/lib/SPIRV/libSPIRV/SPIRVModule.h
++++ b/lib/SPIRV/libSPIRV/SPIRVModule.h
+@@ -416,6 +416,10 @@ public:
+     return TranslationOpts.isSPIRVAllowUnknownIntrinsicsEnabled();
+   }
+ 
++  bool shouldReplaceLLVMFmulAddWithOpenCLMad() const noexcept {
++    return TranslationOpts.shouldReplaceLLVMFmulAddWithOpenCLMad();
++  }
++
+   SPIRVExtInstSetKind getDebugInfoEIS() const {
+     switch (TranslationOpts.getDebugInfoEIS()) {
+     case DebugInfoEIS::SPIRV_Debug:
+diff --git a/test/llvm.fmuladd.ll b/test/llvm.fmuladd.ll
+index aab9b86..5f63454 100644
+--- a/test/llvm.fmuladd.ll
++++ b/test/llvm.fmuladd.ll
+@@ -1,12 +1,25 @@
+ ; RUN: llvm-as %s -o %t.bc
+-; RUN: llvm-spirv %t.bc -spirv-text -o - | FileCheck %s
+-; RUN: llvm-spirv %t.bc -o %t.spv
+-; RUN: spirv-val %t.spv
+-
+-; CHECK-NOT: llvm.fmuladd
+-
+-; CHECK: TypeFloat [[f32:[0-9]+]] 32
+-; CHECK: TypeFloat [[f64:[0-9]+]] 64
++; RUN: llvm-spirv %t.bc -o %default.spv
++; RUN: llvm-spirv %t.bc --spirv-replace-fmuladd-with-ocl-mad=true -o %replace.spv
++; RUN: llvm-spirv %t.bc --spirv-replace-fmuladd-with-ocl-mad=false -o %break.spv
++; RUN: spirv-val %replace.spv
++; RUN: spirv-val %break.spv
++; RUN: llvm-spirv %default.spv -to-text -o - | FileCheck %s --check-prefixes=COMMON,REPLACE
++; RUN: llvm-spirv %replace.spv -to-text -o - | FileCheck %s --check-prefixes=COMMON,REPLACE
++; RUN: llvm-spirv %break.spv -to-text -o - | FileCheck %s --check-prefixes=COMMON,BREAK
++
++; COMMON-NOT: llvm.fmuladd
++
++; COMMON: TypeFloat [[f32:[0-9]+]] 32
++; COMMON: TypeFloat [[f64:[0-9]+]] 64
++;
++; REPLACE: ExtInst [[f32]] {{[0-9]+}} {{[0-9]+}} mad
++; REPLACE: ExtInst [[f64]] {{[0-9]+}} {{[0-9]+}} mad
++;
++; BREAK: FMul [[f32]] [[mul32:[0-9]+]] {{[0-9]+}} {{[0-9]+}}
++; BREAK-NEXT: FAdd [[f32]] {{[0-9]+}} [[mul32]] {{[0-9]+}}
++; BREAK: FMul [[f64]] [[mul64:[0-9]+]] {{[0-9]+}} {{[0-9]+}}
++; BREAK-NEXT: FAdd [[f64]] {{[0-9]+}} [[mul64]] {{[0-9]+}}
+ 
+ target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
+ target triple = "spir64"
+@@ -15,11 +28,7 @@ target triple = "spir64"
+ define spir_func void @foo(float %a, float %b, float %c, double %x, double %y, double %z) #0 {
+ entry:
+   %0 = call float @llvm.fmuladd.f32(float %a, float %b, float %c)
+-; CHECK: FMul [[f32]] [[mul32:[0-9]+]] {{[0-9]+}} {{[0-9]+}}
+-; CHECK-NEXT: FAdd [[f32]] {{[0-9]+}} [[mul32]] {{[0-9]+}}
+   %1 = call double @llvm.fmuladd.f64(double %x, double %y, double %z)
+-; CHECK: FMul [[f64]] [[mul64:[0-9]+]] {{[0-9]+}} {{[0-9]+}}
+-; CHECK-NEXT: FAdd [[f64]] {{[0-9]+}} [[mul64]] {{[0-9]+}}
+ ret void
+ }
+ 
+diff --git a/tools/llvm-spirv/llvm-spirv.cpp b/tools/llvm-spirv/llvm-spirv.cpp
+index b6f1fe4..1ca8acc 100644
+--- a/tools/llvm-spirv/llvm-spirv.cpp
++++ b/tools/llvm-spirv/llvm-spirv.cpp
+@@ -156,6 +156,12 @@ static cl::opt<SPIRV::DebugInfoEIS> DebugEIS(
+                    "extended instruction set. This version of SPIR-V debug "
+                    "info format is compatible with the SPIRV-Tools")));
+ 
++static cl::opt<bool> SPIRVReplaceLLVMFmulAddWithOpenCLMad(
++    "spirv-replace-fmuladd-with-ocl-mad",
++    cl::desc("Allow replacement of llvm.fmuladd.* intrinsic with OpenCL mad "
++             "instruction from OpenCL extended instruction set"),
++    cl::init(true));
++
+ static std::string removeExt(const std::string &FileName) {
+   size_t Pos = FileName.find_last_of(".");
+   if (Pos != std::string::npos)
+@@ -339,6 +345,16 @@ int main(int Ac, char **Av) {
+     }
+   }
+ 
++  if (SPIRVReplaceLLVMFmulAddWithOpenCLMad.getNumOccurrences() != 0) {
++    if (IsReverse) {
++      errs() << "Note: --spirv-replace-fmuladd-with-ocl-mad option ignored as "
++                "it only affects translation from LLVM IR to SPIR-V";
++    } else {
++      Opts.setReplaceLLVMFmulAddWithOpenCLMad(
++          SPIRVReplaceLLVMFmulAddWithOpenCLMad);
++    }
++  }
++
+   if (DebugEIS.getNumOccurrences() != 0) {
+     if (IsReverse) {
+       errs() << "Note: --spirv-debug-info-version option ignored as it only "
+-- 
+2.17.1
+