Bundle operands to specify denormal modes

llvm/docs/LangRef.rst

@@ -3084,7 +3084,10 @@ floating-point control modes and the treatment of status bits respectively.
 
 An operand bundle tagged with "fp.control" contains information about the
 control modes used for the operation execution. Operands specified in this
-bundle represent particular options. Currently, only rounding mode is supported.
+bundle represent particular options. The following modes are supported:
+
+* rounding mode,
+* denormal behavior.
 
 Rounding mode is represented by a metadata string value, which specifies the
 the mode used for the operation evaluation. Possible values are:
@@ -3103,6 +3106,19 @@ rounding rounding mode is taken from the control register (dynamic rounding).
 In the particular case of :ref:`default floating-point environment <floatenv>`,
 the operation uses rounding to nearest, ties to even.
 
+Denormal behavior defines whether denormal values are flushed to zero during
+the call's execution. This behavior is specified separately for input and
+output values. Such specification is a string, which starts with
+"denorm.in=" or "denorm.out=" respectively. The remainder of the string should
+be one of the values:
+
+::
+
+    ``"ieee"`` - preserve denormals,
+    ``"zero"`` - flush to +0.0 or -0.0 depending on value sign,
+    ``"pzero"`` - flush to +0.0,
+    ``"dyn"`` - concrete mode is read from some register.
+
 An operand bundle tagged with "fp.except" may be associated with operations
 that can read or write floating-point exception flags. It contains a single
 metadata string value, which can have one of the following values:

llvm/include/llvm/ADT/FloatingPointMode.h

@@ -234,6 +234,39 @@ void DenormalMode::print(raw_ostream &OS) const {
   OS << denormalModeKindName(Output) << ',' << denormalModeKindName(Input);
 }
 
+/// If the specified string represents denormal mode as used in operand bundles,
+/// returns the corresponding mode.
+inline std::optional<DenormalMode::DenormalModeKind>
+parseDenormalKindFromOperandBundle(StringRef Str) {
+  if (Str == "ieee")
+    return DenormalMode::IEEE;
+  if (Str == "zero")
+    return DenormalMode::PreserveSign;
+  if (Str == "pzero")
+    return DenormalMode::PositiveZero;
+  if (Str == "dyn")
+    return DenormalMode::Dynamic;
+  return std::nullopt;
+}
+
+/// Converts the specified denormal mode into string suitable for use in an
+/// operand bundle.
+inline std::optional<StringRef>
+printDenormalForOperandBundle(DenormalMode::DenormalModeKind Mode) {
+  switch (Mode) {
+  case DenormalMode::IEEE:
+    return "ieee";
+  case DenormalMode::PreserveSign:
+    return "zero";
+  case DenormalMode::PositiveZero:
+    return "pzero";
+  case DenormalMode::Dynamic:
+    return "dyn";
+  default:
+    return std::nullopt;
+  }
+}
+
 /// Floating-point class tests, supported by 'is_fpclass' intrinsic. Actual
 /// test may be an OR combination of basic tests.
 enum FPClassTest : unsigned {

llvm/include/llvm/IR/InstrTypes.h

@@ -1092,12 +1092,24 @@ template <typename InputTy> class OperandBundleDefT {
 using OperandBundleDef = OperandBundleDefT<Value *>;
 using ConstOperandBundleDef = OperandBundleDefT<const Value *>;
 
+std::optional<StringRef> getBundleOperandByPrefix(OperandBundleUse Bundle,
+                                                  StringRef Prefix);
+void addOperandToBundleTag(LLVMContext &Ctx,
+                           SmallVectorImpl<OperandBundleDef> &Bundles,
+                           StringRef Tag, size_t PrefixSize, StringRef Val);
+
 void addFPRoundingBundle(LLVMContext &Ctx,
                          SmallVectorImpl<OperandBundleDef> &Bundles,
                          RoundingMode Rounding);
 void addFPExceptionBundle(LLVMContext &Ctx,
                           SmallVectorImpl<OperandBundleDef> &Bundles,
                           fp::ExceptionBehavior Except);
+void addFPInputDenormBundle(LLVMContext &Ctx,
+                            SmallVectorImpl<OperandBundleDef> &Bundles,
+                            DenormalMode::DenormalModeKind Mode);
+void addFPOutputDenormBundle(LLVMContext &Ctx,
+                             SmallVectorImpl<OperandBundleDef> &Bundles,
+                             DenormalMode::DenormalModeKind Mode);
 
 //===----------------------------------------------------------------------===//
 //                               CallBase Class
@@ -2171,6 +2183,15 @@ class CallBase : public Instruction {
   /// Return exception behavior specified for this call.
   fp::ExceptionBehavior getExceptionBehavior() const;
 
+  /// Return input denormal mode specified by operand bundles.
+  DenormalMode::DenormalModeKind getInputDenormMode() const;
+
+  /// Return output denormal mode specified by operand bundles.
+  DenormalMode::DenormalModeKind getOutputDenormMode() const;
+
+  /// Return input and output denormal modes specified by operand bundles.
+  DenormalMode getDenormMode() const;
+
   /// Used to keep track of an operand bundle.  See the main comment on
   /// OperandBundleUser above.
   struct BundleOpInfo {

llvm/lib/Analysis/ConstantFolding.cpp

@@ -1354,6 +1354,11 @@ static ConstantFP *flushDenormalConstantFP(ConstantFP *CFP,
   if (!APF.isDenormal())
     return CFP;
 
+  if (auto *CB = dyn_cast<CallBase>(Inst)) {
+    auto Mode = IsOutput ? CB->getOutputDenormMode() : CB->getInputDenormMode();
+    return flushDenormalConstant(CFP->getType(), APF, Mode);
+  }
+
   DenormalMode Mode = getInstrDenormalMode(Inst, CFP->getType());
   return flushDenormalConstant(CFP->getType(), APF,
                                IsOutput ? Mode.Output : Mode.Input);
@@ -2976,12 +2981,20 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
   }
 
   if (const auto *Op1 = dyn_cast<ConstantFP>(Operands[0])) {
-    const APFloat &Op1V = Op1->getValueAPF();
+    ConstantFP *Op1F =
+        flushDenormalConstantFP(const_cast<ConstantFP *>(Op1), Call, false);
+    if (!Op1F)
+      return nullptr;
+    const APFloat &Op1V = Op1F->getValueAPF();
 
     if (const auto *Op2 = dyn_cast<ConstantFP>(Operands[1])) {
       if (Op2->getType() != Op1->getType())
         return nullptr;
-      const APFloat &Op2V = Op2->getValueAPF();
+      ConstantFP *Op2F =
+          flushDenormalConstantFP(const_cast<ConstantFP *>(Op2), Call, false);
+      if (!Op2F)
+        return nullptr;
+      const APFloat &Op2V = Op2F->getValueAPF();
 
       if (const auto *ConstrIntr =
               dyn_cast_if_present<ConstrainedFPIntrinsic>(Call)) {
@@ -3011,8 +3024,11 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
           return evaluateCompare(Op1V, Op2V, ConstrIntr);
         }
         if (mayFoldConstrained(const_cast<ConstrainedFPIntrinsic *>(ConstrIntr),
-                               St))
-          return ConstantFP::get(Ty->getContext(), Res);
+                               St)) {
+          auto DenormOut = Call->getOutputDenormMode();
+          DenormalMode::DenormalModeKind Mode = DenormOut;
+          return flushDenormalConstant(Op2->getType(), Res, Mode);
+        }
         return nullptr;
       }
 

llvm/lib/IR/Instructions.cpp

@@ -628,13 +628,17 @@ bool CallBase::hasClobberingOperandBundles() const {
 RoundingMode CallBase::getRoundingMode() const {
   // Try reading rounding mode from FP bundle.
   std::optional<RoundingMode> RM;
-  if (auto RoundingBundle = getOperandBundle(LLVMContext::OB_fp_control)) {
-    Value *V = RoundingBundle->Inputs.front();
-    Metadata *MD = cast<MetadataAsValue>(V)->getMetadata();
-    RM = convertBundleToRoundingMode(cast<MDString>(MD)->getString());
+  if (auto ControlBundle = getOperandBundle(LLVMContext::OB_fp_control)) {
+    for (auto &U : ControlBundle->Inputs) {
+      Value *V = U.get();
+      if (auto *MDV = dyn_cast<MetadataAsValue>(V)) {
+        Metadata *MD = MDV->getMetadata();
+        if (auto *MDS = dyn_cast<MDString>(MD))
+          if (auto RM = convertBundleToRoundingMode(MDS->getString()))
+            return *RM;
+      }
+    }
   }
-  if (RM)
-    return *RM;
 
   // If this is a constrained intrinsic, get rounding mode from its metadata
   // arguments.
@@ -678,6 +682,71 @@ fp::ExceptionBehavior CallBase::getExceptionBehavior() const {
   return fp::ebIgnore;
 }
 
+DenormalMode::DenormalModeKind CallBase::getInputDenormMode() const {
+  if (auto InDenormBundle = getOperandBundle(LLVMContext::OB_fp_control)) {
+    auto DenormOperand =
+        getBundleOperandByPrefix(*InDenormBundle, "denorm.in=");
+    if (DenormOperand) {
+      if (auto Mode = parseDenormalKindFromOperandBundle(*DenormOperand))
+        return *Mode;
+    } else {
+      return DenormalMode::IEEE;
+    }
+  }
+
+  if (!getParent())
+    return DenormalMode::IEEE;
+  const Function *F = getFunction();
+  if (!F)
+    return DenormalMode::IEEE;
+
+  Type *Ty = nullptr;
+  for (auto &A : args())
+    if (auto *T = A.get()->getType(); T->isFPOrFPVectorTy()) {
+      Ty = T;
+      break;
+    }
+  assert(Ty && "Some input argument must be of floating-point type");
+
+  Ty = Ty->getScalarType();
+  return F->getDenormalMode(Ty->getFltSemantics()).Input;
+}
+
+DenormalMode::DenormalModeKind CallBase::getOutputDenormMode() const {
+  if (auto InDenormBundle = getOperandBundle(LLVMContext::OB_fp_control)) {
+    auto DenormOperand =
+        getBundleOperandByPrefix(*InDenormBundle, "denorm.out=");
+    if (DenormOperand) {
+      if (auto Mode = parseDenormalKindFromOperandBundle(*DenormOperand))
+        return *Mode;
+    } else {
+      return DenormalMode::IEEE;
+    }
+  }
+
+  if (!getParent())
+    return DenormalMode::IEEE;
+  const Function *F = getFunction();
+  if (!F)
+    return DenormalMode::IEEE;
+
+  Type *Ty = getType();
+  assert(Ty->isFPOrFPVectorTy() && "Unexpected output type");
+
+  Ty = Ty->getScalarType();
+  return F->getDenormalMode(Ty->getFltSemantics()).Output;
+}
+
+DenormalMode CallBase::getDenormMode() const {
+  auto InputMode = getInputDenormMode();
+  auto OutputMode = getOutputDenormMode();
+  if (!InputMode)
+    InputMode = DenormalMode::IEEE;
+  if (!OutputMode)
+    OutputMode = DenormalMode::IEEE;
+  return DenormalMode(OutputMode, InputMode);
+}
+
 MemoryEffects CallBase::getFloatingPointMemoryEffects() const {
   if (Intrinsic::ID IntrID = getIntrinsicID())
     if (const BasicBlock *BB = getParent())
@@ -794,6 +863,69 @@ bool CallBase::hasArgumentWithAdditionalReturnCaptureComponents() const {
   return false;
 }
 
+std::optional<StringRef> llvm::getBundleOperandByPrefix(OperandBundleUse Bundle,
+                                                      StringRef Prefix) {
+  for (const auto &Item : Bundle.Inputs) {
+    Metadata *MD = cast<MetadataAsValue>(Item.get())->getMetadata();
+    if (const auto *MDS = dyn_cast<MDString>(MD)) {
+      StringRef Str = MDS->getString();
+      if (Str.consume_front(Prefix))
+        return Str;
+    }
+  }
+  return std::nullopt;
+}
+
+void llvm::addOperandToBundleTag(LLVMContext &Ctx,
+                               SmallVectorImpl<OperandBundleDef> &Bundles,
+                               StringRef Tag, size_t PrefixSize,
+                               StringRef Val) {
+  assert(PrefixSize > 0 && "Unexpected prefix size");
+  assert(PrefixSize < Val.size() && "Invalid prefix size");
+  StringRef Prefix = Val.take_front(PrefixSize);
+
+  // Find a bundle with the specified tag.
+  OperandBundleDef *Bundle = nullptr;
+  for (OperandBundleDef &OB : Bundles) {
+    if (OB.getTag() == Tag) {
+      Bundle = &OB;
+      break;
+    }
+  }
+
+  // If no such bundle are found, create new one with the single value.
+  if (!Bundle) {
+    auto *MStr = MDString::get(Ctx, Val);
+    auto *MD = MetadataAsValue::get(Ctx, MStr);
+    SmallVector<Value *, 1> BundleValues(1, MD);
+    Bundles.emplace_back(Tag.str(), BundleValues);
+    return;
+  }
+
+  // If the bundle is found, search its values for those started with the given
+  // prefix.
+  SmallVector<Value *, 4> Values(Bundle->inputs());
+  for (auto i = Values.begin(), e = Values.end(); i != e; ++i) {
+    if (auto *MV = dyn_cast<MetadataAsValue>(*i)) {
+      if (auto *MD = dyn_cast<MDString>(MV->getMetadata())) {
+        StringRef Str = MD->getString();
+        if (Str == Val)
+          // Already in the values.
+          return;
+        if (Str.starts_with(Prefix)) {
+          Values.erase(i);
+          break;
+        }
+      }
+    }
+  }
+
+  auto *ValMD = MDString::get(Ctx, Val);
+  auto *MD = MetadataAsValue::get(Ctx, ValMD);
+  Values.push_back(MD);
+  Bundles.emplace_back("fp.control", Values);
+}
+
 void llvm::addFPRoundingBundle(LLVMContext &Ctx,
                                SmallVectorImpl<OperandBundleDef> &Bundles,
                                RoundingMode Rounding) {
@@ -814,6 +946,30 @@ void llvm::addFPExceptionBundle(LLVMContext &Ctx,
   Bundles.emplace_back("fp.except", EB);
 }
 
+void llvm::addFPInputDenormBundle(LLVMContext &Ctx,
+                                  SmallVectorImpl<OperandBundleDef> &Bundles,
+                                  DenormalMode::DenormalModeKind Mode) {
+  std::optional<StringRef> DenormValue = printDenormalForOperandBundle(Mode);
+  if (!DenormValue)
+    return;
+  std::string Prefix = "denorm.in=";
+  std::string DenormItem = Prefix + DenormValue->str();
+
+  addOperandToBundleTag(Ctx, Bundles, "fp.control", Prefix.size(), DenormItem);
+}
+
+void llvm::addFPOutputDenormBundle(LLVMContext &Ctx,
+                                   SmallVectorImpl<OperandBundleDef> &Bundles,
+                                   DenormalMode::DenormalModeKind Mode) {
+  std::optional<StringRef> DenormValue = printDenormalForOperandBundle(Mode);
+  if (!DenormValue)
+    return;
+  std::string Prefix = "denorm.out=";
+  std::string DenormItem = Prefix + DenormValue->str();
+
+  addOperandToBundleTag(Ctx, Bundles, "fp.control", Prefix.size(), DenormItem);
+}
+
 //===----------------------------------------------------------------------===//
 //                        CallInst Implementation
 //===----------------------------------------------------------------------===//

llvm/lib/IR/Verifier.cpp

@@ -3833,6 +3833,8 @@ void Verifier::visitCallBase(CallBase &Call) {
       Check(!FoundFpeControlBundle, "Multiple \"fp.control\" operand bundles",
             Call);
       bool FoundRoundingMode = false;
+      bool FoundInDenormalMode = false;
+      bool FoundOutDenormalMode = false;
       for (auto &U : BU.Inputs) {
         Value *V = U.get();
         Check(isa<MetadataAsValue>(V),
@@ -3847,6 +3849,18 @@ void Verifier::visitCallBase(CallBase &Call) {
           Check(!FoundRoundingMode, "Rounding mode is specified more that once",
                 Call);
           FoundRoundingMode = true;
+        } else if (Item.consume_front("denorm.in=")) {
+          Check(!FoundInDenormalMode,
+                "Input denormal mode is specified more that once", Call);
+          FoundInDenormalMode = true;
+          Check(parseDenormalKindFromOperandBundle(Item),
+                "Invalid input denormal mode", Call);
+        } else if (Item.consume_front("denorm.out=")) {
+          Check(!FoundOutDenormalMode,
+                "Output denormal mode is specified more that once", Call);
+          FoundOutDenormalMode = true;
+          Check(parseDenormalKindFromOperandBundle(Item),
+                "Invalid output denormal mode", Call);
         } else {
           CheckFailed("Unrecognized value in \"fp.control\" bundle operand",
                       Call);