Add constant-folding for unary NVVM intrinsics #141233

LewisCrawford · 2025-05-23T13:24:25Z

Add support for constant-folding numerous NVVM unary arithmetic intrinsics (including f, d, and ftz_f variants):

nvvm.ceil.*
nvvm.cos.approx.*
nvvm.ex2.approx.*
nvvm.fabs.*
nvvm.floor.*
nvvm.lg2.approx.*
nvvm.rcp.*
nvvm.round.*
nvvm.rsqrt.approx.*
nvvm.saturate.*
nvvm.sin.approx.*
nvvm.sqrt.f
nvvm.sqrt.rn.*
nvvm.sqrt.approx.*

llvmbot · 2025-05-23T13:25:00Z

@llvm/pr-subscribers-llvm-analysis

@llvm/pr-subscribers-backend-nvptx

Author: Lewis Crawford (LewisCrawford)

Changes

Add support for constant-folding numerous NVVM unary arithmetic intrinsics (including f, d, and ftz_f variants):

nvvm.ceil.*
nvvm.cos.approx.*
nvvm.ex2.approx.*
nvvm.fabs.*
nvvm.floor.*
nvvm.lg2.approx.*
nvvm.rcp.*
nvvm.round.*
nvvm.rsqrt.approx.*
nvvm.saturate.*
nvvm.sin.approx.*
nvvm.sqrt.f
nvvm.sqrt.rn.*
nvvm.sqrt.approx.*

Patch is 45.78 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/141233.diff

3 Files Affected:

(modified) llvm/include/llvm/IR/NVVMIntrinsicUtils.h (+122)
(modified) llvm/lib/Analysis/ConstantFolding.cpp (+207-4)
(added) llvm/test/Transforms/InstSimplify/const-fold-nvvm-unary-arithmetic.ll (+1035)

diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index ce794e2573637..394430f5e5629 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -334,6 +334,128 @@ inline bool FMinFMaxIsXorSignAbs(Intrinsic::ID IntrinsicID) {
   return false;
 }
 
+inline bool UnaryMathIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID) {
+  switch (IntrinsicID) {
+  case Intrinsic::nvvm_ceil_ftz_f:
+  case Intrinsic::nvvm_cos_approx_ftz_f:
+  case Intrinsic::nvvm_ex2_approx_ftz_f:
+  case Intrinsic::nvvm_fabs_ftz_f:
+  case Intrinsic::nvvm_floor_ftz_f:
+  case Intrinsic::nvvm_lg2_approx_ftz_f:
+  case Intrinsic::nvvm_round_ftz_f:
+  case Intrinsic::nvvm_rsqrt_approx_ftz_d:
+  case Intrinsic::nvvm_rsqrt_approx_ftz_f:
+  case Intrinsic::nvvm_saturate_ftz_f:
+  case Intrinsic::nvvm_sin_approx_ftz_f:
+  case Intrinsic::nvvm_sqrt_rn_ftz_f:
+  case Intrinsic::nvvm_sqrt_approx_ftz_f:
+    return true;
+  case Intrinsic::nvvm_ceil_f:
+  case Intrinsic::nvvm_ceil_d:
+  case Intrinsic::nvvm_cos_approx_f:
+  case Intrinsic::nvvm_ex2_approx_d:
+  case Intrinsic::nvvm_ex2_approx_f:
+  case Intrinsic::nvvm_fabs_d:
+  case Intrinsic::nvvm_fabs_f:
+  case Intrinsic::nvvm_floor_f:
+  case Intrinsic::nvvm_floor_d:
+  case Intrinsic::nvvm_lg2_approx_d:
+  case Intrinsic::nvvm_lg2_approx_f:
+  case Intrinsic::nvvm_round_f:
+  case Intrinsic::nvvm_round_d:
+  case Intrinsic::nvvm_rsqrt_approx_d:
+  case Intrinsic::nvvm_rsqrt_approx_f:
+  case Intrinsic::nvvm_saturate_d:
+  case Intrinsic::nvvm_saturate_f:
+  case Intrinsic::nvvm_sin_approx_f:
+  case Intrinsic::nvvm_sqrt_f:
+  case Intrinsic::nvvm_sqrt_rn_d:
+  case Intrinsic::nvvm_sqrt_rn_f:
+  case Intrinsic::nvvm_sqrt_approx_f:
+    return false;
+  }
+  llvm_unreachable("Checking FTZ flag for invalid unary intrinsic");
+  return false;
+}
+
+inline bool RCPShouldFTZ(Intrinsic::ID IntrinsicID) {
+  switch (IntrinsicID) {
+  case Intrinsic::nvvm_rcp_rm_ftz_f:
+  case Intrinsic::nvvm_rcp_rn_ftz_f:
+  case Intrinsic::nvvm_rcp_rp_ftz_f:
+  case Intrinsic::nvvm_rcp_rz_ftz_f:
+  case Intrinsic::nvvm_rcp_approx_ftz_f:
+  case Intrinsic::nvvm_rcp_approx_ftz_d:
+    return true;
+  case Intrinsic::nvvm_rcp_rm_d:
+  case Intrinsic::nvvm_rcp_rm_f:
+  case Intrinsic::nvvm_rcp_rn_d:
+  case Intrinsic::nvvm_rcp_rn_f:
+  case Intrinsic::nvvm_rcp_rp_d:
+  case Intrinsic::nvvm_rcp_rp_f:
+  case Intrinsic::nvvm_rcp_rz_d:
+  case Intrinsic::nvvm_rcp_rz_f:
+    return false;
+  }
+  llvm_unreachable("Checking FTZ flag for invalid rcp intrinsic");
+  return false;
+}
+
+inline APFloat::roundingMode GetRCPRoundingMode(Intrinsic::ID IntrinsicID) {
+  switch (IntrinsicID) {
+  case Intrinsic::nvvm_rcp_rm_f:
+  case Intrinsic::nvvm_rcp_rm_d:
+  case Intrinsic::nvvm_rcp_rm_ftz_f:
+    return APFloat::rmTowardNegative;
+
+  case Intrinsic::nvvm_rcp_approx_ftz_f:
+  case Intrinsic::nvvm_rcp_approx_ftz_d:
+  case Intrinsic::nvvm_rcp_rn_f:
+  case Intrinsic::nvvm_rcp_rn_d:
+  case Intrinsic::nvvm_rcp_rn_ftz_f:
+    return APFloat::rmNearestTiesToEven;
+
+  case Intrinsic::nvvm_rcp_rp_f:
+  case Intrinsic::nvvm_rcp_rp_d:
+  case Intrinsic::nvvm_rcp_rp_ftz_f:
+    return APFloat::rmNearestTiesToEven;
+
+  case Intrinsic::nvvm_rcp_rz_f:
+  case Intrinsic::nvvm_rcp_rz_d:
+  case Intrinsic::nvvm_rcp_rz_ftz_f:
+    return APFloat::rmTowardZero;
+  }
+  llvm_unreachable("Checking rounding mode for invalid rcp intrinsic");
+  return APFloat::roundingMode::Invalid;
+}
+
+inline bool RCPIsApprox(Intrinsic::ID IntrinsicID) {
+  switch (IntrinsicID) {
+  case Intrinsic::nvvm_rcp_approx_ftz_f:
+  case Intrinsic::nvvm_rcp_approx_ftz_d:
+    return true;
+
+  case Intrinsic::nvvm_rcp_rm_f:
+  case Intrinsic::nvvm_rcp_rm_d:
+  case Intrinsic::nvvm_rcp_rm_ftz_f:
+
+  case Intrinsic::nvvm_rcp_rn_f:
+  case Intrinsic::nvvm_rcp_rn_d:
+  case Intrinsic::nvvm_rcp_rn_ftz_f:
+
+  case Intrinsic::nvvm_rcp_rp_f:
+  case Intrinsic::nvvm_rcp_rp_d:
+  case Intrinsic::nvvm_rcp_rp_ftz_f:
+
+  case Intrinsic::nvvm_rcp_rz_f:
+  case Intrinsic::nvvm_rcp_rz_d:
+  case Intrinsic::nvvm_rcp_rz_ftz_f:
+    return false;
+  }
+  llvm_unreachable("Checking approx flag for invalid rcp intrinsic");
+  return false;
+}
+
 } // namespace nvvm
 } // namespace llvm
 #endif // LLVM_IR_NVVMINTRINSICUTILS_H
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 412a0e8979193..d8d3b2fad8bd1 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1791,6 +1791,69 @@ bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
   case Intrinsic::nearbyint:
   case Intrinsic::rint:
   case Intrinsic::canonicalize:
+
+  // NVVM math intrinsics:
+  case Intrinsic::nvvm_ceil_d:
+  case Intrinsic::nvvm_ceil_f:
+  case Intrinsic::nvvm_ceil_ftz_f:
+
+  case Intrinsic::nvvm_cos_approx_f:
+  case Intrinsic::nvvm_cos_approx_ftz_f:
+
+  case Intrinsic::nvvm_ex2_approx_d:
+  case Intrinsic::nvvm_ex2_approx_f:
+  case Intrinsic::nvvm_ex2_approx_ftz_f:
+
+  case Intrinsic::nvvm_fabs_d:
+  case Intrinsic::nvvm_fabs_f:
+  case Intrinsic::nvvm_fabs_ftz_f:
+
+  case Intrinsic::nvvm_floor_d:
+  case Intrinsic::nvvm_floor_f:
+  case Intrinsic::nvvm_floor_ftz_f:
+
+  case Intrinsic::nvvm_lg2_approx_d:
+  case Intrinsic::nvvm_lg2_approx_f:
+  case Intrinsic::nvvm_lg2_approx_ftz_f:
+
+  case Intrinsic::nvvm_rcp_rm_d:
+  case Intrinsic::nvvm_rcp_rm_f:
+  case Intrinsic::nvvm_rcp_rm_ftz_f:
+  case Intrinsic::nvvm_rcp_rn_d:
+  case Intrinsic::nvvm_rcp_rn_f:
+  case Intrinsic::nvvm_rcp_rn_ftz_f:
+  case Intrinsic::nvvm_rcp_rp_d:
+  case Intrinsic::nvvm_rcp_rp_f:
+  case Intrinsic::nvvm_rcp_rp_ftz_f:
+  case Intrinsic::nvvm_rcp_rz_d:
+  case Intrinsic::nvvm_rcp_rz_f:
+  case Intrinsic::nvvm_rcp_rz_ftz_f:
+  case Intrinsic::nvvm_rcp_approx_ftz_d:
+  case Intrinsic::nvvm_rcp_approx_ftz_f:
+
+  case Intrinsic::nvvm_round_d:
+  case Intrinsic::nvvm_round_f:
+  case Intrinsic::nvvm_round_ftz_f:
+
+  case Intrinsic::nvvm_rsqrt_approx_d:
+  case Intrinsic::nvvm_rsqrt_approx_f:
+  case Intrinsic::nvvm_rsqrt_approx_ftz_d:
+  case Intrinsic::nvvm_rsqrt_approx_ftz_f:
+
+  case Intrinsic::nvvm_saturate_d:
+  case Intrinsic::nvvm_saturate_f:
+  case Intrinsic::nvvm_saturate_ftz_f:
+
+  case Intrinsic::nvvm_sin_approx_f:
+  case Intrinsic::nvvm_sin_approx_ftz_f:
+
+  case Intrinsic::nvvm_sqrt_f:
+  case Intrinsic::nvvm_sqrt_rn_d:
+  case Intrinsic::nvvm_sqrt_rn_f:
+  case Intrinsic::nvvm_sqrt_rn_ftz_f:
+  case Intrinsic::nvvm_sqrt_approx_f:
+  case Intrinsic::nvvm_sqrt_approx_ftz_f:
+
   // Constrained intrinsics can be folded if FP environment is known
   // to compiler.
   case Intrinsic::experimental_constrained_fma:
@@ -1944,16 +2007,23 @@ static const APFloat FTZPreserveSign(const APFloat &V) {
   return V;
 }
 
-Constant *ConstantFoldFP(double (*NativeFP)(double), const APFloat &V,
-                         Type *Ty) {
+Constant *ConstantFoldFP(double (*NativeFP)(double), const APFloat &V, Type *Ty,
+                         bool ShouldFTZPreservingSign = false) {
   llvm_fenv_clearexcept();
-  double Result = NativeFP(V.convertToDouble());
+  auto Input = ShouldFTZPreservingSign ? FTZPreserveSign(V) : V;
+  double Result = NativeFP(Input.convertToDouble());
   if (llvm_fenv_testexcept()) {
     llvm_fenv_clearexcept();
     return nullptr;
   }
 
-  return GetConstantFoldFPValue(Result, Ty);
+  Constant *Output = GetConstantFoldFPValue(Result, Ty);
+  if (ShouldFTZPreservingSign) {
+    const auto *CFP = static_cast<ConstantFP *>(Output);
+    return ConstantFP::get(Ty->getContext(),
+                           FTZPreserveSign(CFP->getValueAPF()));
+  }
+  return Output;
 }
 
 #if defined(HAS_IEE754_FLOAT128) && defined(HAS_LOGF128)
@@ -2524,6 +2594,139 @@ static Constant *ConstantFoldScalarCall1(StringRef Name,
         return ConstantFoldFP(cosh, APF, Ty);
       case Intrinsic::sqrt:
         return ConstantFoldFP(sqrt, APF, Ty);
+
+      // NVVM Intrinsics:
+      case Intrinsic::nvvm_ceil_ftz_f:
+      case Intrinsic::nvvm_ceil_f:
+      case Intrinsic::nvvm_ceil_d:
+        return ConstantFoldFP(ceil, APF, Ty,
+                              nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID));
+
+      case Intrinsic::nvvm_cos_approx_ftz_f:
+      case Intrinsic::nvvm_cos_approx_f:
+        return ConstantFoldFP(cos, APF, Ty,
+                              nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID));
+
+      case Intrinsic::nvvm_ex2_approx_ftz_f:
+      case Intrinsic::nvvm_ex2_approx_d:
+      case Intrinsic::nvvm_ex2_approx_f:
+        return ConstantFoldFP(exp2, APF, Ty,
+                              nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID));
+
+      case Intrinsic::nvvm_fabs_ftz_f:
+      case Intrinsic::nvvm_fabs_d:
+      case Intrinsic::nvvm_fabs_f:
+        return ConstantFoldFP(fabs, APF, Ty,
+                              nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID));
+
+      case Intrinsic::nvvm_floor_ftz_f:
+      case Intrinsic::nvvm_floor_f:
+      case Intrinsic::nvvm_floor_d:
+        return ConstantFoldFP(floor, APF, Ty,
+                              nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID));
+
+      case Intrinsic::nvvm_lg2_approx_ftz_f:
+      case Intrinsic::nvvm_lg2_approx_d:
+      case Intrinsic::nvvm_lg2_approx_f: {
+        if (APF.isNegative() || APF.isZero())
+          return nullptr;
+        return ConstantFoldFP(log2, APF, Ty,
+                              nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID));
+      }
+
+      case Intrinsic::nvvm_rcp_rm_ftz_f:
+      case Intrinsic::nvvm_rcp_rn_ftz_f:
+      case Intrinsic::nvvm_rcp_rp_ftz_f:
+      case Intrinsic::nvvm_rcp_rz_ftz_f:
+      case Intrinsic::nvvm_rcp_approx_ftz_f:
+      case Intrinsic::nvvm_rcp_approx_ftz_d:
+      case Intrinsic::nvvm_rcp_rm_d:
+      case Intrinsic::nvvm_rcp_rm_f:
+      case Intrinsic::nvvm_rcp_rn_d:
+      case Intrinsic::nvvm_rcp_rn_f:
+      case Intrinsic::nvvm_rcp_rp_d:
+      case Intrinsic::nvvm_rcp_rp_f:
+      case Intrinsic::nvvm_rcp_rz_d:
+      case Intrinsic::nvvm_rcp_rz_f: {
+        APFloat::roundingMode RoundMode = nvvm::GetRCPRoundingMode(IntrinsicID);
+        bool IsApprox = nvvm::RCPIsApprox(IntrinsicID);
+        bool IsFTZ = nvvm::RCPShouldFTZ(IntrinsicID);
+
+        auto Denominator = IsFTZ ? FTZPreserveSign(APF) : APF;
+        if (IsApprox && Denominator.isZero()) {
+          // According to the PTX spec, approximate rcp should return infinity
+          // with the same sign as the denominator when dividing by 0.
+          APFloat Inf = APFloat::getInf(APF.getSemantics(), APF.isNegative());
+          return ConstantFP::get(Ty->getContext(), Inf);
+        }
+        APFloat Res = APFloat::getOne(APF.getSemantics());
+        APFloat::opStatus Status = Res.divide(Denominator, RoundMode);
+
+        if (Status == APFloat::opOK || Status == APFloat::opInexact) {
+          if (IsFTZ)
+            Res = FTZPreserveSign(Res);
+          return ConstantFP::get(Ty->getContext(), Res);
+        }
+        return nullptr;
+      }
+
+      case Intrinsic::nvvm_round_ftz_f:
+      case Intrinsic::nvvm_round_f:
+      case Intrinsic::nvvm_round_d:
+        return ConstantFoldFP(round, APF, Ty,
+                              nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID));
+
+      case Intrinsic::nvvm_rsqrt_approx_ftz_d:
+      case Intrinsic::nvvm_rsqrt_approx_ftz_f:
+      case Intrinsic::nvvm_rsqrt_approx_d:
+      case Intrinsic::nvvm_rsqrt_approx_f: {
+        bool IsFTZ = nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID);
+        auto V = IsFTZ ? FTZPreserveSign(APF) : APF;
+        APFloat SqrtV(sqrt(V.convertToDouble()));
+
+        bool lost;
+        SqrtV.convert(APF.getSemantics(), APFloat::rmNearestTiesToEven, &lost);
+
+        APFloat Res = APFloat::getOne(APF.getSemantics());
+        Res.divide(SqrtV, APFloat::rmNearestTiesToEven);
+
+        // We do not need to flush the output for ftz because it is impossible
+        // for 1/sqrt(x) to be a denormal value. If x is the largest fp value,
+        // sqrt(x) will be a number with the exponent approximately halved and
+        // the reciprocal of that number can't be small enough to be denormal.
+        return ConstantFP::get(Ty->getContext(), Res);
+      }
+
+      case Intrinsic::nvvm_saturate_ftz_f:
+      case Intrinsic::nvvm_saturate_d:
+      case Intrinsic::nvvm_saturate_f: {
+        bool IsFTZ = nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID);
+        auto V = IsFTZ ? FTZPreserveSign(APF) : APF;
+        if (V.isNegative() || V.isZero() || V.isNaN())
+          return ConstantFP::getZero(Ty);
+        APFloat One = APFloat::getOne(APF.getSemantics());
+        if (V > One)
+          return ConstantFP::get(Ty->getContext(), One);
+        return ConstantFP::get(Ty->getContext(), APF);
+      }
+
+      case Intrinsic::nvvm_sin_approx_ftz_f:
+      case Intrinsic::nvvm_sin_approx_f:
+        return ConstantFoldFP(sin, APF, Ty,
+                              nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID));
+
+      case Intrinsic::nvvm_sqrt_rn_ftz_f:
+      case Intrinsic::nvvm_sqrt_approx_ftz_f:
+      case Intrinsic::nvvm_sqrt_f:
+      case Intrinsic::nvvm_sqrt_rn_d:
+      case Intrinsic::nvvm_sqrt_rn_f:
+      case Intrinsic::nvvm_sqrt_approx_f:
+        if (APF.isNegative())
+          return nullptr;
+        return ConstantFoldFP(sqrt, APF, Ty,
+                              nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID));
+
+      // AMDGCN Intrinsics:
       case Intrinsic::amdgcn_cos:
       case Intrinsic::amdgcn_sin: {
         double V = getValueAsDouble(Op);
diff --git a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-unary-arithmetic.ll b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-unary-arithmetic.ll
new file mode 100644
index 0000000000000..51563975be233
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-unary-arithmetic.ll
@@ -0,0 +1,1035 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=instsimplify -march=nvptx64 -S | FileCheck %s
+
+; Test constant-folding for various NVVM unary arithmetic intrinsics.
+
+;###############################################################
+;#                          Ceil                               #
+;###############################################################
+
+define double @test_ceil_d_1_25() {
+; CHECK-LABEL: define double @test_ceil_d_1_25() {
+; CHECK-NEXT:    ret double 2.000000e+00
+;
+  %res = call double @llvm.nvvm.ceil.d(double 1.25)
+  ret double %res
+}
+
+define float @test_ceil_f_1_25() {
+; CHECK-LABEL: define float @test_ceil_f_1_25() {
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %res = call float @llvm.nvvm.ceil.f(float 1.25)
+  ret float %res
+}
+
+define float @test_ceil_ftz_f_1_25() {
+; CHECK-LABEL: define float @test_ceil_ftz_f_1_25() {
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %res = call float @llvm.nvvm.ceil.ftz.f(float 1.25)
+  ret float %res
+}
+
+define double @test_ceil_d_pos_subnorm() {
+; CHECK-LABEL: define double @test_ceil_d_pos_subnorm() {
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = call double @llvm.nvvm.ceil.d(double 0x380FFFFFC0000000)
+  ret double %res
+}
+
+define float @test_ceil_f_pos_subnorm() {
+; CHECK-LABEL: define float @test_ceil_f_pos_subnorm() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.ceil.f(float 0x380FFFFFC0000000)
+  ret float %res
+}
+
+define float @test_ceil_ftz_f_pos_subnorm() {
+; CHECK-LABEL: define float @test_ceil_ftz_f_pos_subnorm() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.ceil.ftz.f(float 0x380FFFFFC0000000)
+  ret float %res
+}
+
+;###############################################################
+;#                        Cos Approx                           #
+;###############################################################
+
+define float @test_cos_approx_f_1_25() {
+; CHECK-LABEL: define float @test_cos_approx_f_1_25() {
+; CHECK-NEXT:    ret float 0x3FD42E3DE0000000
+;
+  %res = call float @llvm.nvvm.cos.approx.f(float 1.25)
+  ret float %res
+}
+
+define float @test_cos_approx_ftz_f_1_25() {
+; CHECK-LABEL: define float @test_cos_approx_ftz_f_1_25() {
+; CHECK-NEXT:    ret float 0x3FD42E3DE0000000
+;
+  %res = call float @llvm.nvvm.cos.approx.ftz.f(float 1.25)
+  ret float %res
+}
+
+define float @test_cos_approx_f_pos_subnorm() {
+; CHECK-LABEL: define float @test_cos_approx_f_pos_subnorm() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.cos.approx.f(float 0x380FFFFFC0000000)
+  ret float %res
+}
+
+define float @test_cos_approx_ftz_f_pos_subnorm() {
+; CHECK-LABEL: define float @test_cos_approx_ftz_f_pos_subnorm() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.cos.approx.ftz.f(float 0x380FFFFFC0000000)
+  ret float %res
+}
+
+;###############################################################
+;#                        Ex2 Approx                           #
+;###############################################################
+
+define double @test_ex2_approx_d_1_25() {
+; CHECK-LABEL: define double @test_ex2_approx_d_1_25() {
+; CHECK-NEXT:    ret double 0x400306FE0A31B715
+;
+  %res = call double @llvm.nvvm.ex2.approx.d(double 1.25)
+  ret double %res
+}
+
+define float @test_ex2_approx_f_1_25() {
+; CHECK-LABEL: define float @test_ex2_approx_f_1_25() {
+; CHECK-NEXT:    ret float 0x400306FE00000000
+;
+  %res = call float @llvm.nvvm.ex2.approx.f(float 1.25)
+  ret float %res
+}
+
+define float @test_ex2_approx_ftz_f_1_25() {
+; CHECK-LABEL: define float @test_ex2_approx_ftz_f_1_25() {
+; CHECK-NEXT:    ret float 0x400306FE00000000
+;
+  %res = call float @llvm.nvvm.ex2.approx.ftz.f(float 1.25)
+  ret float %res
+}
+
+define double @test_ex2_approx_d_pos_subnorm() {
+; CHECK-LABEL: define double @test_ex2_approx_d_pos_subnorm() {
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = call double @llvm.nvvm.ex2.approx.d(double 0x380FFFFFC0000000)
+  ret double %res
+}
+
+define float @test_ex2_approx_f_pos_subnorm() {
+; CHECK-LABEL: define float @test_ex2_approx_f_pos_subnorm() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.ex2.approx.f(float 0x380FFFFFC0000000)
+  ret float %res
+}
+
+define float @test_ex2_approx_ftz_f_pos_subnorm() {
+; CHECK-LABEL: define float @test_ex2_approx_ftz_f_pos_subnorm() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.ex2.approx.ftz.f(float 0x380FFFFFC0000000)
+  ret float %res
+}
+
+;###############################################################
+;#                          FAbs                               #
+;###############################################################
+define double @test_fabs_d_neg_1_5() {
+; CHECK-LABEL: define double @test_fabs_d_neg_1_5() {
+; CHECK-NEXT:    ret double 1.500000e+00
+;
+  %res = call double @llvm.nvvm.fabs.d(double -1.5)
+  ret double %res
+}
+
+define float @test_fabs_f_neg_1_5() {
+; CHECK-LABEL: define float @test_fabs_f_neg_1_5() {
+; CHECK-NEXT:    ret float 1.500000e+00
+;
+  %res = call float @llvm.nvvm.fabs.f(float -1.5)
+  ret float %res
+}
+
+define float @test_fabs_ftz_f_neg_1_5() {
+; CHECK-LABEL: define float @test_fabs_ftz_f_neg_1_5() {
+; CHECK-NEXT:    ret float 1.500000e+00
+;
+  %res = call float @llvm.nvvm.fabs.ftz.f(float -1.5)
+  ret float %res
+}
+
+define double @test_fabs_d_1_25() {
+; CHECK-LABEL: define double @test_fabs_d_1_25() {
+; CHECK-NEXT:    ret double 1.250000e+00
+;
+  %res = call double @llvm.nvvm.fabs.d(double 1.25)
+  ret double %res
+}
+
+define float @test_fabs_f_1_25() {
+; CHECK-LABEL: define float @test_fabs_f_1_25() {
+; CHECK-NEXT:    ret float 1.250000e+00
+;
+  %res = call float @llvm.nvvm.fabs.f(float 1.25)
+  ret float %res
+}
+
+define float @test_fabs_ftz_f_1_25() {
+; CHECK-LABEL: define float @test_fabs_ftz_f_1_25() {
+; CHECK-NEXT:    ret float 1.250000e+00
+;
+  %res = call float @llvm.nvvm.fabs.ftz.f(float 1.25)
+  ret float %res
+}
+
+define double @test_fabs_d_neg_subnorm() {
+; CHECK-LABEL: define double @test_fabs_d_neg_subnorm() {
+; CHECK-NEXT:    ret double 0x380FFFFFC0000000
+;
+  %res = call double @llvm.nvvm.fabs.d(double 0xB80FFFFFC0000000)
+  ret double %res
+}
+
+define float @test_fabs_f_neg_subnorm() {
+; CHECK-LABEL: define float @test_fabs_f_neg_subnorm() {
+; CHECK-NEXT:    ret floa...
[truncated]

llvmbot · 2025-05-23T13:25:00Z

@llvm/pr-subscribers-llvm-ir

Author: Lewis Crawford (LewisCrawford)

Changes

Add support for constant-folding numerous NVVM unary arithmetic intrinsics (including f, d, and ftz_f variants):

nvvm.ceil.*
nvvm.cos.approx.*
nvvm.ex2.approx.*
nvvm.fabs.*
nvvm.floor.*
nvvm.lg2.approx.*
nvvm.rcp.*
nvvm.round.*
nvvm.rsqrt.approx.*
nvvm.saturate.*
nvvm.sin.approx.*
nvvm.sqrt.f
nvvm.sqrt.rn.*
nvvm.sqrt.approx.*

Patch is 45.78 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/141233.diff

3 Files Affected:

(modified) llvm/include/llvm/IR/NVVMIntrinsicUtils.h (+122)
(modified) llvm/lib/Analysis/ConstantFolding.cpp (+207-4)
(added) llvm/test/Transforms/InstSimplify/const-fold-nvvm-unary-arithmetic.ll (+1035)

diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index ce794e2573637..394430f5e5629 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -334,6 +334,128 @@ inline bool FMinFMaxIsXorSignAbs(Intrinsic::ID IntrinsicID) {
   return false;
 }
 
+inline bool UnaryMathIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID) {
+  switch (IntrinsicID) {
+  case Intrinsic::nvvm_ceil_ftz_f:
+  case Intrinsic::nvvm_cos_approx_ftz_f:
+  case Intrinsic::nvvm_ex2_approx_ftz_f:
+  case Intrinsic::nvvm_fabs_ftz_f:
+  case Intrinsic::nvvm_floor_ftz_f:
+  case Intrinsic::nvvm_lg2_approx_ftz_f:
+  case Intrinsic::nvvm_round_ftz_f:
+  case Intrinsic::nvvm_rsqrt_approx_ftz_d:
+  case Intrinsic::nvvm_rsqrt_approx_ftz_f:
+  case Intrinsic::nvvm_saturate_ftz_f:
+  case Intrinsic::nvvm_sin_approx_ftz_f:
+  case Intrinsic::nvvm_sqrt_rn_ftz_f:
+  case Intrinsic::nvvm_sqrt_approx_ftz_f:
+    return true;
+  case Intrinsic::nvvm_ceil_f:
+  case Intrinsic::nvvm_ceil_d:
+  case Intrinsic::nvvm_cos_approx_f:
+  case Intrinsic::nvvm_ex2_approx_d:
+  case Intrinsic::nvvm_ex2_approx_f:
+  case Intrinsic::nvvm_fabs_d:
+  case Intrinsic::nvvm_fabs_f:
+  case Intrinsic::nvvm_floor_f:
+  case Intrinsic::nvvm_floor_d:
+  case Intrinsic::nvvm_lg2_approx_d:
+  case Intrinsic::nvvm_lg2_approx_f:
+  case Intrinsic::nvvm_round_f:
+  case Intrinsic::nvvm_round_d:
+  case Intrinsic::nvvm_rsqrt_approx_d:
+  case Intrinsic::nvvm_rsqrt_approx_f:
+  case Intrinsic::nvvm_saturate_d:
+  case Intrinsic::nvvm_saturate_f:
+  case Intrinsic::nvvm_sin_approx_f:
+  case Intrinsic::nvvm_sqrt_f:
+  case Intrinsic::nvvm_sqrt_rn_d:
+  case Intrinsic::nvvm_sqrt_rn_f:
+  case Intrinsic::nvvm_sqrt_approx_f:
+    return false;
+  }
+  llvm_unreachable("Checking FTZ flag for invalid unary intrinsic");
+  return false;
+}
+
+inline bool RCPShouldFTZ(Intrinsic::ID IntrinsicID) {
+  switch (IntrinsicID) {
+  case Intrinsic::nvvm_rcp_rm_ftz_f:
+  case Intrinsic::nvvm_rcp_rn_ftz_f:
+  case Intrinsic::nvvm_rcp_rp_ftz_f:
+  case Intrinsic::nvvm_rcp_rz_ftz_f:
+  case Intrinsic::nvvm_rcp_approx_ftz_f:
+  case Intrinsic::nvvm_rcp_approx_ftz_d:
+    return true;
+  case Intrinsic::nvvm_rcp_rm_d:
+  case Intrinsic::nvvm_rcp_rm_f:
+  case Intrinsic::nvvm_rcp_rn_d:
+  case Intrinsic::nvvm_rcp_rn_f:
+  case Intrinsic::nvvm_rcp_rp_d:
+  case Intrinsic::nvvm_rcp_rp_f:
+  case Intrinsic::nvvm_rcp_rz_d:
+  case Intrinsic::nvvm_rcp_rz_f:
+    return false;
+  }
+  llvm_unreachable("Checking FTZ flag for invalid rcp intrinsic");
+  return false;
+}
+
+inline APFloat::roundingMode GetRCPRoundingMode(Intrinsic::ID IntrinsicID) {
+  switch (IntrinsicID) {
+  case Intrinsic::nvvm_rcp_rm_f:
+  case Intrinsic::nvvm_rcp_rm_d:
+  case Intrinsic::nvvm_rcp_rm_ftz_f:
+    return APFloat::rmTowardNegative;
+
+  case Intrinsic::nvvm_rcp_approx_ftz_f:
+  case Intrinsic::nvvm_rcp_approx_ftz_d:
+  case Intrinsic::nvvm_rcp_rn_f:
+  case Intrinsic::nvvm_rcp_rn_d:
+  case Intrinsic::nvvm_rcp_rn_ftz_f:
+    return APFloat::rmNearestTiesToEven;
+
+  case Intrinsic::nvvm_rcp_rp_f:
+  case Intrinsic::nvvm_rcp_rp_d:
+  case Intrinsic::nvvm_rcp_rp_ftz_f:
+    return APFloat::rmNearestTiesToEven;
+
+  case Intrinsic::nvvm_rcp_rz_f:
+  case Intrinsic::nvvm_rcp_rz_d:
+  case Intrinsic::nvvm_rcp_rz_ftz_f:
+    return APFloat::rmTowardZero;
+  }
+  llvm_unreachable("Checking rounding mode for invalid rcp intrinsic");
+  return APFloat::roundingMode::Invalid;
+}
+
+inline bool RCPIsApprox(Intrinsic::ID IntrinsicID) {
+  switch (IntrinsicID) {
+  case Intrinsic::nvvm_rcp_approx_ftz_f:
+  case Intrinsic::nvvm_rcp_approx_ftz_d:
+    return true;
+
+  case Intrinsic::nvvm_rcp_rm_f:
+  case Intrinsic::nvvm_rcp_rm_d:
+  case Intrinsic::nvvm_rcp_rm_ftz_f:
+
+  case Intrinsic::nvvm_rcp_rn_f:
+  case Intrinsic::nvvm_rcp_rn_d:
+  case Intrinsic::nvvm_rcp_rn_ftz_f:
+
+  case Intrinsic::nvvm_rcp_rp_f:
+  case Intrinsic::nvvm_rcp_rp_d:
+  case Intrinsic::nvvm_rcp_rp_ftz_f:
+
+  case Intrinsic::nvvm_rcp_rz_f:
+  case Intrinsic::nvvm_rcp_rz_d:
+  case Intrinsic::nvvm_rcp_rz_ftz_f:
+    return false;
+  }
+  llvm_unreachable("Checking approx flag for invalid rcp intrinsic");
+  return false;
+}
+
 } // namespace nvvm
 } // namespace llvm
 #endif // LLVM_IR_NVVMINTRINSICUTILS_H
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 412a0e8979193..d8d3b2fad8bd1 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1791,6 +1791,69 @@ bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
   case Intrinsic::nearbyint:
   case Intrinsic::rint:
   case Intrinsic::canonicalize:
+
+  // NVVM math intrinsics:
+  case Intrinsic::nvvm_ceil_d:
+  case Intrinsic::nvvm_ceil_f:
+  case Intrinsic::nvvm_ceil_ftz_f:
+
+  case Intrinsic::nvvm_cos_approx_f:
+  case Intrinsic::nvvm_cos_approx_ftz_f:
+
+  case Intrinsic::nvvm_ex2_approx_d:
+  case Intrinsic::nvvm_ex2_approx_f:
+  case Intrinsic::nvvm_ex2_approx_ftz_f:
+
+  case Intrinsic::nvvm_fabs_d:
+  case Intrinsic::nvvm_fabs_f:
+  case Intrinsic::nvvm_fabs_ftz_f:
+
+  case Intrinsic::nvvm_floor_d:
+  case Intrinsic::nvvm_floor_f:
+  case Intrinsic::nvvm_floor_ftz_f:
+
+  case Intrinsic::nvvm_lg2_approx_d:
+  case Intrinsic::nvvm_lg2_approx_f:
+  case Intrinsic::nvvm_lg2_approx_ftz_f:
+
+  case Intrinsic::nvvm_rcp_rm_d:
+  case Intrinsic::nvvm_rcp_rm_f:
+  case Intrinsic::nvvm_rcp_rm_ftz_f:
+  case Intrinsic::nvvm_rcp_rn_d:
+  case Intrinsic::nvvm_rcp_rn_f:
+  case Intrinsic::nvvm_rcp_rn_ftz_f:
+  case Intrinsic::nvvm_rcp_rp_d:
+  case Intrinsic::nvvm_rcp_rp_f:
+  case Intrinsic::nvvm_rcp_rp_ftz_f:
+  case Intrinsic::nvvm_rcp_rz_d:
+  case Intrinsic::nvvm_rcp_rz_f:
+  case Intrinsic::nvvm_rcp_rz_ftz_f:
+  case Intrinsic::nvvm_rcp_approx_ftz_d:
+  case Intrinsic::nvvm_rcp_approx_ftz_f:
+
+  case Intrinsic::nvvm_round_d:
+  case Intrinsic::nvvm_round_f:
+  case Intrinsic::nvvm_round_ftz_f:
+
+  case Intrinsic::nvvm_rsqrt_approx_d:
+  case Intrinsic::nvvm_rsqrt_approx_f:
+  case Intrinsic::nvvm_rsqrt_approx_ftz_d:
+  case Intrinsic::nvvm_rsqrt_approx_ftz_f:
+
+  case Intrinsic::nvvm_saturate_d:
+  case Intrinsic::nvvm_saturate_f:
+  case Intrinsic::nvvm_saturate_ftz_f:
+
+  case Intrinsic::nvvm_sin_approx_f:
+  case Intrinsic::nvvm_sin_approx_ftz_f:
+
+  case Intrinsic::nvvm_sqrt_f:
+  case Intrinsic::nvvm_sqrt_rn_d:
+  case Intrinsic::nvvm_sqrt_rn_f:
+  case Intrinsic::nvvm_sqrt_rn_ftz_f:
+  case Intrinsic::nvvm_sqrt_approx_f:
+  case Intrinsic::nvvm_sqrt_approx_ftz_f:
+
   // Constrained intrinsics can be folded if FP environment is known
   // to compiler.
   case Intrinsic::experimental_constrained_fma:
@@ -1944,16 +2007,23 @@ static const APFloat FTZPreserveSign(const APFloat &V) {
   return V;
 }
 
-Constant *ConstantFoldFP(double (*NativeFP)(double), const APFloat &V,
-                         Type *Ty) {
+Constant *ConstantFoldFP(double (*NativeFP)(double), const APFloat &V, Type *Ty,
+                         bool ShouldFTZPreservingSign = false) {
   llvm_fenv_clearexcept();
-  double Result = NativeFP(V.convertToDouble());
+  auto Input = ShouldFTZPreservingSign ? FTZPreserveSign(V) : V;
+  double Result = NativeFP(Input.convertToDouble());
   if (llvm_fenv_testexcept()) {
     llvm_fenv_clearexcept();
     return nullptr;
   }
 
-  return GetConstantFoldFPValue(Result, Ty);
+  Constant *Output = GetConstantFoldFPValue(Result, Ty);
+  if (ShouldFTZPreservingSign) {
+    const auto *CFP = static_cast<ConstantFP *>(Output);
+    return ConstantFP::get(Ty->getContext(),
+                           FTZPreserveSign(CFP->getValueAPF()));
+  }
+  return Output;
 }
 
 #if defined(HAS_IEE754_FLOAT128) && defined(HAS_LOGF128)
@@ -2524,6 +2594,139 @@ static Constant *ConstantFoldScalarCall1(StringRef Name,
         return ConstantFoldFP(cosh, APF, Ty);
       case Intrinsic::sqrt:
         return ConstantFoldFP(sqrt, APF, Ty);
+
+      // NVVM Intrinsics:
+      case Intrinsic::nvvm_ceil_ftz_f:
+      case Intrinsic::nvvm_ceil_f:
+      case Intrinsic::nvvm_ceil_d:
+        return ConstantFoldFP(ceil, APF, Ty,
+                              nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID));
+
+      case Intrinsic::nvvm_cos_approx_ftz_f:
+      case Intrinsic::nvvm_cos_approx_f:
+        return ConstantFoldFP(cos, APF, Ty,
+                              nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID));
+
+      case Intrinsic::nvvm_ex2_approx_ftz_f:
+      case Intrinsic::nvvm_ex2_approx_d:
+      case Intrinsic::nvvm_ex2_approx_f:
+        return ConstantFoldFP(exp2, APF, Ty,
+                              nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID));
+
+      case Intrinsic::nvvm_fabs_ftz_f:
+      case Intrinsic::nvvm_fabs_d:
+      case Intrinsic::nvvm_fabs_f:
+        return ConstantFoldFP(fabs, APF, Ty,
+                              nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID));
+
+      case Intrinsic::nvvm_floor_ftz_f:
+      case Intrinsic::nvvm_floor_f:
+      case Intrinsic::nvvm_floor_d:
+        return ConstantFoldFP(floor, APF, Ty,
+                              nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID));
+
+      case Intrinsic::nvvm_lg2_approx_ftz_f:
+      case Intrinsic::nvvm_lg2_approx_d:
+      case Intrinsic::nvvm_lg2_approx_f: {
+        if (APF.isNegative() || APF.isZero())
+          return nullptr;
+        return ConstantFoldFP(log2, APF, Ty,
+                              nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID));
+      }
+
+      case Intrinsic::nvvm_rcp_rm_ftz_f:
+      case Intrinsic::nvvm_rcp_rn_ftz_f:
+      case Intrinsic::nvvm_rcp_rp_ftz_f:
+      case Intrinsic::nvvm_rcp_rz_ftz_f:
+      case Intrinsic::nvvm_rcp_approx_ftz_f:
+      case Intrinsic::nvvm_rcp_approx_ftz_d:
+      case Intrinsic::nvvm_rcp_rm_d:
+      case Intrinsic::nvvm_rcp_rm_f:
+      case Intrinsic::nvvm_rcp_rn_d:
+      case Intrinsic::nvvm_rcp_rn_f:
+      case Intrinsic::nvvm_rcp_rp_d:
+      case Intrinsic::nvvm_rcp_rp_f:
+      case Intrinsic::nvvm_rcp_rz_d:
+      case Intrinsic::nvvm_rcp_rz_f: {
+        APFloat::roundingMode RoundMode = nvvm::GetRCPRoundingMode(IntrinsicID);
+        bool IsApprox = nvvm::RCPIsApprox(IntrinsicID);
+        bool IsFTZ = nvvm::RCPShouldFTZ(IntrinsicID);
+
+        auto Denominator = IsFTZ ? FTZPreserveSign(APF) : APF;
+        if (IsApprox && Denominator.isZero()) {
+          // According to the PTX spec, approximate rcp should return infinity
+          // with the same sign as the denominator when dividing by 0.
+          APFloat Inf = APFloat::getInf(APF.getSemantics(), APF.isNegative());
+          return ConstantFP::get(Ty->getContext(), Inf);
+        }
+        APFloat Res = APFloat::getOne(APF.getSemantics());
+        APFloat::opStatus Status = Res.divide(Denominator, RoundMode);
+
+        if (Status == APFloat::opOK || Status == APFloat::opInexact) {
+          if (IsFTZ)
+            Res = FTZPreserveSign(Res);
+          return ConstantFP::get(Ty->getContext(), Res);
+        }
+        return nullptr;
+      }
+
+      case Intrinsic::nvvm_round_ftz_f:
+      case Intrinsic::nvvm_round_f:
+      case Intrinsic::nvvm_round_d:
+        return ConstantFoldFP(round, APF, Ty,
+                              nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID));
+
+      case Intrinsic::nvvm_rsqrt_approx_ftz_d:
+      case Intrinsic::nvvm_rsqrt_approx_ftz_f:
+      case Intrinsic::nvvm_rsqrt_approx_d:
+      case Intrinsic::nvvm_rsqrt_approx_f: {
+        bool IsFTZ = nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID);
+        auto V = IsFTZ ? FTZPreserveSign(APF) : APF;
+        APFloat SqrtV(sqrt(V.convertToDouble()));
+
+        bool lost;
+        SqrtV.convert(APF.getSemantics(), APFloat::rmNearestTiesToEven, &lost);
+
+        APFloat Res = APFloat::getOne(APF.getSemantics());
+        Res.divide(SqrtV, APFloat::rmNearestTiesToEven);
+
+        // We do not need to flush the output for ftz because it is impossible
+        // for 1/sqrt(x) to be a denormal value. If x is the largest fp value,
+        // sqrt(x) will be a number with the exponent approximately halved and
+        // the reciprocal of that number can't be small enough to be denormal.
+        return ConstantFP::get(Ty->getContext(), Res);
+      }
+
+      case Intrinsic::nvvm_saturate_ftz_f:
+      case Intrinsic::nvvm_saturate_d:
+      case Intrinsic::nvvm_saturate_f: {
+        bool IsFTZ = nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID);
+        auto V = IsFTZ ? FTZPreserveSign(APF) : APF;
+        if (V.isNegative() || V.isZero() || V.isNaN())
+          return ConstantFP::getZero(Ty);
+        APFloat One = APFloat::getOne(APF.getSemantics());
+        if (V > One)
+          return ConstantFP::get(Ty->getContext(), One);
+        return ConstantFP::get(Ty->getContext(), APF);
+      }
+
+      case Intrinsic::nvvm_sin_approx_ftz_f:
+      case Intrinsic::nvvm_sin_approx_f:
+        return ConstantFoldFP(sin, APF, Ty,
+                              nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID));
+
+      case Intrinsic::nvvm_sqrt_rn_ftz_f:
+      case Intrinsic::nvvm_sqrt_approx_ftz_f:
+      case Intrinsic::nvvm_sqrt_f:
+      case Intrinsic::nvvm_sqrt_rn_d:
+      case Intrinsic::nvvm_sqrt_rn_f:
+      case Intrinsic::nvvm_sqrt_approx_f:
+        if (APF.isNegative())
+          return nullptr;
+        return ConstantFoldFP(sqrt, APF, Ty,
+                              nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID));
+
+      // AMDGCN Intrinsics:
       case Intrinsic::amdgcn_cos:
       case Intrinsic::amdgcn_sin: {
         double V = getValueAsDouble(Op);
diff --git a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-unary-arithmetic.ll b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-unary-arithmetic.ll
new file mode 100644
index 0000000000000..51563975be233
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-unary-arithmetic.ll
@@ -0,0 +1,1035 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=instsimplify -march=nvptx64 -S | FileCheck %s
+
+; Test constant-folding for various NVVM unary arithmetic intrinsics.
+
+;###############################################################
+;#                          Ceil                               #
+;###############################################################
+
+define double @test_ceil_d_1_25() {
+; CHECK-LABEL: define double @test_ceil_d_1_25() {
+; CHECK-NEXT:    ret double 2.000000e+00
+;
+  %res = call double @llvm.nvvm.ceil.d(double 1.25)
+  ret double %res
+}
+
+define float @test_ceil_f_1_25() {
+; CHECK-LABEL: define float @test_ceil_f_1_25() {
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %res = call float @llvm.nvvm.ceil.f(float 1.25)
+  ret float %res
+}
+
+define float @test_ceil_ftz_f_1_25() {
+; CHECK-LABEL: define float @test_ceil_ftz_f_1_25() {
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %res = call float @llvm.nvvm.ceil.ftz.f(float 1.25)
+  ret float %res
+}
+
+define double @test_ceil_d_pos_subnorm() {
+; CHECK-LABEL: define double @test_ceil_d_pos_subnorm() {
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = call double @llvm.nvvm.ceil.d(double 0x380FFFFFC0000000)
+  ret double %res
+}
+
+define float @test_ceil_f_pos_subnorm() {
+; CHECK-LABEL: define float @test_ceil_f_pos_subnorm() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.ceil.f(float 0x380FFFFFC0000000)
+  ret float %res
+}
+
+define float @test_ceil_ftz_f_pos_subnorm() {
+; CHECK-LABEL: define float @test_ceil_ftz_f_pos_subnorm() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.ceil.ftz.f(float 0x380FFFFFC0000000)
+  ret float %res
+}
+
+;###############################################################
+;#                        Cos Approx                           #
+;###############################################################
+
+define float @test_cos_approx_f_1_25() {
+; CHECK-LABEL: define float @test_cos_approx_f_1_25() {
+; CHECK-NEXT:    ret float 0x3FD42E3DE0000000
+;
+  %res = call float @llvm.nvvm.cos.approx.f(float 1.25)
+  ret float %res
+}
+
+define float @test_cos_approx_ftz_f_1_25() {
+; CHECK-LABEL: define float @test_cos_approx_ftz_f_1_25() {
+; CHECK-NEXT:    ret float 0x3FD42E3DE0000000
+;
+  %res = call float @llvm.nvvm.cos.approx.ftz.f(float 1.25)
+  ret float %res
+}
+
+define float @test_cos_approx_f_pos_subnorm() {
+; CHECK-LABEL: define float @test_cos_approx_f_pos_subnorm() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.cos.approx.f(float 0x380FFFFFC0000000)
+  ret float %res
+}
+
+define float @test_cos_approx_ftz_f_pos_subnorm() {
+; CHECK-LABEL: define float @test_cos_approx_ftz_f_pos_subnorm() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.cos.approx.ftz.f(float 0x380FFFFFC0000000)
+  ret float %res
+}
+
+;###############################################################
+;#                        Ex2 Approx                           #
+;###############################################################
+
+define double @test_ex2_approx_d_1_25() {
+; CHECK-LABEL: define double @test_ex2_approx_d_1_25() {
+; CHECK-NEXT:    ret double 0x400306FE0A31B715
+;
+  %res = call double @llvm.nvvm.ex2.approx.d(double 1.25)
+  ret double %res
+}
+
+define float @test_ex2_approx_f_1_25() {
+; CHECK-LABEL: define float @test_ex2_approx_f_1_25() {
+; CHECK-NEXT:    ret float 0x400306FE00000000
+;
+  %res = call float @llvm.nvvm.ex2.approx.f(float 1.25)
+  ret float %res
+}
+
+define float @test_ex2_approx_ftz_f_1_25() {
+; CHECK-LABEL: define float @test_ex2_approx_ftz_f_1_25() {
+; CHECK-NEXT:    ret float 0x400306FE00000000
+;
+  %res = call float @llvm.nvvm.ex2.approx.ftz.f(float 1.25)
+  ret float %res
+}
+
+define double @test_ex2_approx_d_pos_subnorm() {
+; CHECK-LABEL: define double @test_ex2_approx_d_pos_subnorm() {
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = call double @llvm.nvvm.ex2.approx.d(double 0x380FFFFFC0000000)
+  ret double %res
+}
+
+define float @test_ex2_approx_f_pos_subnorm() {
+; CHECK-LABEL: define float @test_ex2_approx_f_pos_subnorm() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.ex2.approx.f(float 0x380FFFFFC0000000)
+  ret float %res
+}
+
+define float @test_ex2_approx_ftz_f_pos_subnorm() {
+; CHECK-LABEL: define float @test_ex2_approx_ftz_f_pos_subnorm() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.ex2.approx.ftz.f(float 0x380FFFFFC0000000)
+  ret float %res
+}
+
+;###############################################################
+;#                          FAbs                               #
+;###############################################################
+define double @test_fabs_d_neg_1_5() {
+; CHECK-LABEL: define double @test_fabs_d_neg_1_5() {
+; CHECK-NEXT:    ret double 1.500000e+00
+;
+  %res = call double @llvm.nvvm.fabs.d(double -1.5)
+  ret double %res
+}
+
+define float @test_fabs_f_neg_1_5() {
+; CHECK-LABEL: define float @test_fabs_f_neg_1_5() {
+; CHECK-NEXT:    ret float 1.500000e+00
+;
+  %res = call float @llvm.nvvm.fabs.f(float -1.5)
+  ret float %res
+}
+
+define float @test_fabs_ftz_f_neg_1_5() {
+; CHECK-LABEL: define float @test_fabs_ftz_f_neg_1_5() {
+; CHECK-NEXT:    ret float 1.500000e+00
+;
+  %res = call float @llvm.nvvm.fabs.ftz.f(float -1.5)
+  ret float %res
+}
+
+define double @test_fabs_d_1_25() {
+; CHECK-LABEL: define double @test_fabs_d_1_25() {
+; CHECK-NEXT:    ret double 1.250000e+00
+;
+  %res = call double @llvm.nvvm.fabs.d(double 1.25)
+  ret double %res
+}
+
+define float @test_fabs_f_1_25() {
+; CHECK-LABEL: define float @test_fabs_f_1_25() {
+; CHECK-NEXT:    ret float 1.250000e+00
+;
+  %res = call float @llvm.nvvm.fabs.f(float 1.25)
+  ret float %res
+}
+
+define float @test_fabs_ftz_f_1_25() {
+; CHECK-LABEL: define float @test_fabs_ftz_f_1_25() {
+; CHECK-NEXT:    ret float 1.250000e+00
+;
+  %res = call float @llvm.nvvm.fabs.ftz.f(float 1.25)
+  ret float %res
+}
+
+define double @test_fabs_d_neg_subnorm() {
+; CHECK-LABEL: define double @test_fabs_d_neg_subnorm() {
+; CHECK-NEXT:    ret double 0x380FFFFFC0000000
+;
+  %res = call double @llvm.nvvm.fabs.d(double 0xB80FFFFFC0000000)
+  ret double %res
+}
+
+define float @test_fabs_f_neg_subnorm() {
+; CHECK-LABEL: define float @test_fabs_f_neg_subnorm() {
+; CHECK-NEXT:    ret floa...
[truncated]

LewisCrawford · 2025-05-23T13:34:21Z

I would like to land this patch: #140270 before merging this. That patch adds an option to disable constant-folding for FP function calls, and may be useful in the few cases where the device-side execution of these intrinsics must be a bitwise-exact match to the host-side constant-folded version of the results (or when debugging).

Add support for constant-folding numerous NVVM unary arithmetic intrinsics (including f, d, and ftz_f variants): - nvvm.ceil.* - nvvm.cos.approx.* - nvvm.ex2.approx.* - nvvm.fabs.* - nvvm.floor.* - nvvm.lg2.approx.* - nvvm.rcp.* - nvvm.round.* - nvvm.rsqrt.approx.* - nvvm.saturate.* - nvvm.sin.approx.* - nvvm.sqrt.f - nvvm.sqrt.rn.* - nvvm.sqrt.approx.*

Fix the rcp_approx_ftz_d and rsqrt_approx_ftz_d implementations to better match the PTX spec, which states that the inputs and outputs should zero the lower 32-bits of the mantissa.

llvm/include/llvm/IR/NVVMIntrinsicUtils.h

llvm/lib/Analysis/ConstantFolding.cpp

RCP incorrectly folded rcp.rp as rcp.rn due to a bug in NVVMIntrinsicUtils.h.

Remove "return" statements providing default values after llvm_unreachable calls in NVVMIntrinsicUtils.h.

Change ConstantFoldFP to accept an optional DenormalMode parameter, which specifies how denormal inputs and outputs are handled. Avoid handling for invalid/dynamic modes, and allow for flushing to either positive or sign-preserving zero (which may be different for input vs output modes). Allow the NVVM intrinsics to be folded with either the IEEE mode (preserves subnormals), or the sign-preserving-zero mode depending on whether the FTZ variant of the intrinsic is used.

github-actions · 2025-07-02T08:14:03Z

✅ With the latest revision this PR passed the C/C++ code formatter.

LewisCrawford · 2025-07-09T08:42:54Z

@Artem-B @AlexMaclean Does this patch seem ok to merge now?

llvm/lib/Analysis/ConstantFolding.cpp

Artem-B

LGTM with a test suggestion.

Artem-B · 2025-07-16T17:18:42Z

llvm/test/Transforms/InstSimplify/const-fold-nvvm-unary-arithmetic.ll

+;#                          Ceil                               #
+;###############################################################
+
+define double @test_ceil_d_1_25() {


I'd also add test cases for the now-removed approximate intrinsics to verify that we deliberately not const-folding them.

Fix a failing test for constant-folding the nvvm_round intrinsic. The original implementation added in llvm#141233 used a native libm call to the "round" function, but on PPC this produces +0.0 if the input is -0.0, which caused a test failure. This patch updates it to use APFloat functions instead of native libm calls to ensure cross-platform consistency.

Fix a failing test for constant-folding the nvvm_round intrinsic. The original implementation added in #141233 used a native libm call to the "round" function, but on PPC this produces +0.0 if the input is -0.0, which caused a test failure. This patch updates it to use APFloat functions instead of native libm calls to ensure cross-platform consistency.

LewisCrawford · 2025-07-21T16:52:12Z

The lit-test failures for nvvm_round on PPC that this patch caused here: https://lab.llvm.org/buildbot/#/builders/64/builds/4929 should be fixed now by the follow-up patch in #149837 .

Add support for constant-folding numerous NVVM unary arithmetic intrinsics (including f, d, and ftz_f variants): - nvvm.ceil.* - nvvm.fabs.* - nvvm.floor.* - nvvm.rcp.* - nvvm.round.* - nvvm.saturate.* - nvvm.sqrt.f - nvvm.sqrt.rn.*

Fix a failing test for constant-folding the nvvm_round intrinsic. The original implementation added in llvm#141233 used a native libm call to the "round" function, but on PPC this produces +0.0 if the input is -0.0, which caused a test failure. This patch updates it to use APFloat functions instead of native libm calls to ensure cross-platform consistency.

llvmbot added llvm:instcombine Covers the InstCombine, InstSimplify and AggressiveInstCombine passes backend:NVPTX llvm:ir llvm:analysis Includes value tracking, cost tables and constant folding llvm:transforms labels May 23, 2025

LewisCrawford force-pushed the fold_nvvm_math_intrinsics branch from a952b64 to 2338e8c Compare May 23, 2025 13:25

LewisCrawford mentioned this pull request May 23, 2025

[ConstantFolding] Add flag to disable call folding #140270

Merged

LewisCrawford force-pushed the fold_nvvm_math_intrinsics branch from 2338e8c to 9faec17 Compare May 23, 2025 16:20

LewisCrawford requested review from AlexMaclean and Artem-B and removed request for Artem-B May 23, 2025 16:21

LewisCrawford added 2 commits May 30, 2025 12:01

Fix _approx_ftz_d implementations for rcp + rsqrt

d8188fb

Fix the rcp_approx_ftz_d and rsqrt_approx_ftz_d implementations to better match the PTX spec, which states that the inputs and outputs should zero the lower 32-bits of the mantissa.

LewisCrawford force-pushed the fold_nvvm_math_intrinsics branch from a983447 to d8188fb Compare May 30, 2025 12:01

Artem-B reviewed May 30, 2025

View reviewed changes

llvm/include/llvm/IR/NVVMIntrinsicUtils.h Outdated Show resolved Hide resolved

llvm/lib/Analysis/ConstantFolding.cpp Outdated Show resolved Hide resolved

LewisCrawford added 4 commits June 30, 2025 12:07

Merge branch 'main' into fold_nvvm_math_intrinsics

e65eccf

Fix rcp.rp rounding modes

54654e8

RCP incorrectly folded rcp.rp as rcp.rn due to a bug in NVVMIntrinsicUtils.h.

Remove returns after llvm_unreachable

75eb448

Remove "return" statements providing default values after llvm_unreachable calls in NVVMIntrinsicUtils.h.

LewisCrawford requested a review from Artem-B July 2, 2025 08:13

LewisCrawford added 2 commits July 2, 2025 08:15

Fix clang-format error

d96002d

Merge branch 'main' into fold_nvvm_math_intrinsics

387a299

Artem-B reviewed Jul 9, 2025

View reviewed changes

llvm/lib/Analysis/ConstantFolding.cpp Outdated Show resolved Hide resolved

Guard nvvm intrinsic folding with StrictFP check

ca39bbc

Remove all approximate intrinsic folding

9854230

LewisCrawford force-pushed the fold_nvvm_math_intrinsics branch from 91d02a0 to 9854230 Compare July 16, 2025 15:56

Artem-B approved these changes Jul 16, 2025

View reviewed changes

LewisCrawford merged commit fd8ae2c into llvm:main Jul 21, 2025
9 checks passed

LewisCrawford mentioned this pull request Jul 21, 2025

[ConstantFolding] Fix nvvm_round folding on PPC #149837

Merged

This was referenced Jul 23, 2025

test abhinavgaba/llvm-project#2

Closed

Add dataFence plugin interface abhinavgaba/llvm-project#3

Closed

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Add constant-folding for unary NVVM intrinsics #141233

Add constant-folding for unary NVVM intrinsics #141233

Uh oh!

LewisCrawford commented May 23, 2025 •

edited

Loading

Uh oh!

llvmbot commented May 23, 2025 •

edited

Loading

Uh oh!

llvmbot commented May 23, 2025

Uh oh!

LewisCrawford commented May 23, 2025

Uh oh!

Uh oh!

Uh oh!

github-actions bot commented Jul 2, 2025 •

edited

Loading

Uh oh!

LewisCrawford commented Jul 9, 2025

Uh oh!

Uh oh!

Artem-B left a comment

Uh oh!

Artem-B Jul 16, 2025

Uh oh!

Uh oh!

LewisCrawford commented Jul 21, 2025

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

4 participants

Add constant-folding for unary NVVM intrinsics #141233

Add constant-folding for unary NVVM intrinsics #141233

Uh oh!

Conversation

LewisCrawford commented May 23, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

llvmbot commented May 23, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

llvmbot commented May 23, 2025

Uh oh!

LewisCrawford commented May 23, 2025

Uh oh!

Uh oh!

Uh oh!

github-actions bot commented Jul 2, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

LewisCrawford commented Jul 9, 2025

Uh oh!

Uh oh!

Artem-B left a comment

Choose a reason for hiding this comment

Uh oh!

Artem-B Jul 16, 2025

Choose a reason for hiding this comment

Uh oh!

Uh oh!

LewisCrawford commented Jul 21, 2025

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

4 participants

LewisCrawford commented May 23, 2025 •

edited

Loading

llvmbot commented May 23, 2025 •

edited

Loading

github-actions bot commented Jul 2, 2025 •

edited

Loading