Minimal support of floating-point operand bundles

clang/test/CodeGen/strictfp-elementwise-builtins.cpp

@@ -210,7 +210,7 @@ float4 strict_elementwise_rint(float4 a) {
 // CHECK-LABEL: define dso_local noundef <4 x float> @_Z28strict_elementwise_nearbyintDv4_f
 // CHECK-SAME: (<4 x float> noundef [[A:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[ELT_NEARBYINT:%.*]] = tail call <4 x float> @llvm.nearbyint.v4f32(<4 x float> [[A]]) #[[ATTR4]]
+// CHECK-NEXT:    [[ELT_NEARBYINT:%.*]] = tail call <4 x float> @llvm.nearbyint.v4f32(<4 x float> [[A]]) #[[ATTR5:[0-9]+]]
 // CHECK-NEXT:    ret <4 x float> [[ELT_NEARBYINT]]
 //
 float4 strict_elementwise_nearbyint(float4 a) {
@@ -300,7 +300,7 @@ float4 strict_elementwise_atan2(float4 a, float4 b) {
 // CHECK-LABEL: define dso_local noundef <4 x float> @_Z24strict_elementwise_truncDv4_f
 // CHECK-SAME: (<4 x float> noundef [[A:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[ELT_TRUNC:%.*]] = tail call <4 x float> @llvm.trunc.v4f32(<4 x float> [[A]]) #[[ATTR4]]
+// CHECK-NEXT:    [[ELT_TRUNC:%.*]] = tail call <4 x float> @llvm.trunc.v4f32(<4 x float> [[A]]) #[[ATTR5]]
 // CHECK-NEXT:    ret <4 x float> [[ELT_TRUNC]]
 //
 float4 strict_elementwise_trunc(float4 a) {

llvm/docs/LangRef.rst

@@ -3199,6 +3199,57 @@ A "convergencectrl" operand bundle is only valid on a ``convergent`` operation.
 When present, the operand bundle must contain exactly one value of token type.
 See the :doc:`ConvergentOperations` document for details.
 
+.. _ob_fp:
+
+Floating-point Operand Bundles
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+These operand bundles are used for calls that involve floating-point
+operations and interact with :ref:`floating-point environment <floatenv>` or
+depend on floating-point options, such as rounding mode, denormal modes, etc.
+
+An operand bundle tagged with "fp.round" contains information about the
+rounding mode used for the operation execution (the effective rounding mode).
+This mode is represented by a metadata string value and specifies the mode used
+for the operation evaluation. Possible values are:
+
+::
+
+    "towardzero"
+    "tonearest"
+    "upward"
+    "downward"
+    "tonearestaway"
+    "dynamic"
+
+Only one value may be specified. If the "fp.round" bundle is absent, and
+the operation depends on rounding mode, the default behavior is to use the value
+from a control register (dynamic rounding). In the specific case of
+:ref:`default floating-point environment <floatenv>`, the register is assumed to
+set rounding to nearest, ties to even.
+
+An operand bundle tagged with "fp.except" may be associated with operations
+that can raise floating-point exceptions. It contains a single metadata string
+value, which can have one of the following:
+
+::
+
+    "ignore"
+    "strict"
+
+If the argument is "ignore", floating-point exceptions raised by the call are not
+intended to be observed. Optimization transformations may reorder such operations
+or omit them in some cases. For example, if the call result is unused, the call
+may be removed, even if it could raise exceptions. This is the only permitted value
+in the :ref:`default floating-point environment <floatenv>`.
+
+If the argument of "fp.except" is "strict", all transformations must preserve the
+floating-point exception semantics of the original code. Any exception that would
+have been raised by the original code must also be raised by the transformed code
+unless it can be proven unobservable. No new observable floating-point exceptions
+may be introduced. This value may only be used only in functions with
+``strictfp`` attribute.
+
 .. _moduleasm:
 
 Module-Level Inline Assembly
@@ -3949,9 +4000,9 @@ round-to-nearest rounding mode, and subnormals are assumed to be preserved.
 Running LLVM code in an environment where these assumptions are not met
 typically leads to undefined behavior. The ``strictfp`` and ``denormal-fp-math``
 attributes as well as :ref:`Constrained Floating-Point Intrinsics
-<constrainedfp>` can be used to weaken LLVM's assumptions and ensure defined
-behavior in non-default floating-point environments; see their respective
-documentation for details.
+<constrainedfp>` or :ref:`floating-point operand bundles<ob_fp>` can be used to
+weaken LLVM's assumptions and ensure defined behavior in non-default
+floating-point environments; see their respective documentation for details.
 
 .. _floatnan:
 

llvm/docs/ReleaseNotes.md

@@ -67,6 +67,9 @@ Changes to the LLVM IR
   Instead, the `align` attribute should be placed on the pointer (or vector of
   pointers) argument.
 * A `load atomic` may now be used with vector types on x86.
+* Floating-point operand bundles have been added.
+* Calls to floating-point intrinsics can have operand bundles "fp.round" and
+  "fp.except", which specify effective rounding mode and exception behavior.
 
 Changes to LLVM infrastructure
 ------------------------------

llvm/include/llvm/IR/FPEnv.h

@@ -49,20 +49,43 @@ enum ExceptionBehavior : uint8_t {
 /// metadata.
 LLVM_ABI std::optional<RoundingMode> convertStrToRoundingMode(StringRef);
 
+/// Returns a valid RoundingMode enumerator given a string that is used as
+/// rounding mode specifier in operand bundles.
+std::optional<RoundingMode> convertBundleToRoundingMode(StringRef);
+
 /// For any RoundingMode enumerator, returns a string valid as input in
 /// constrained intrinsic rounding mode metadata.
 LLVM_ABI std::optional<StringRef> convertRoundingModeToStr(RoundingMode);
 
+/// For any RoundingMode enumerator, returns a string to be used in operand
+/// bundles.
+std::optional<StringRef> convertRoundingModeToBundle(RoundingMode);
+
 /// Returns a valid ExceptionBehavior enumerator when given a string
 /// valid as input in constrained intrinsic exception behavior metadata.
 LLVM_ABI std::optional<fp::ExceptionBehavior>
     convertStrToExceptionBehavior(StringRef);
 
+/// Returns a valid ExceptionBehavior enumerator given a string from the operand
+/// bundle argument.
+std::optional<fp::ExceptionBehavior>
+    convertBundleToExceptionBehavior(StringRef);
+
 /// For any ExceptionBehavior enumerator, returns a string valid as
 /// input in constrained intrinsic exception behavior metadata.
 LLVM_ABI std::optional<StringRef>
     convertExceptionBehaviorToStr(fp::ExceptionBehavior);
 
+/// Return string representing the given exception behavior for use in operand
+/// bundles
+std::optional<StringRef>
+    convertExceptionBehaviorToBundle(fp::ExceptionBehavior);
+
+inline raw_ostream &operator<<(raw_ostream &OS, fp::ExceptionBehavior EB) {
+  OS << convertExceptionBehaviorToBundle(EB).value_or("invalid");
+  return OS;
+}
+
 /// Returns true if the exception handling behavior and rounding mode
 /// match what is used in the default floating point environment.
 inline bool isDefaultFPEnvironment(fp::ExceptionBehavior EB, RoundingMode RM) {

llvm/include/llvm/IR/FloatingPointOps.def

@@ -0,0 +1,30 @@
+//===- llvm/IR/FloatingPointOps.def - FP intrinsics -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines set of intrinsics, which are classified as floating-point operations.
+//
+//===----------------------------------------------------------------------===//
+
+// Describes floating-point operation.
+// Arguments of the entries are:
+// N - intrinsic function name,
+// D - DAG node corresponding to the intrinsic.
+#ifndef FUNCTION
+#define FUNCTION(N,D)
+#endif
+
+// Describes floating-point operation, which lowers to STRICT_* nodes in DAG.
+#ifndef LEGACY_DAG
+#define LEGACY_DAG(N,D) FUNCTION(N,D)
+#endif
+
+LEGACY_DAG(nearbyint,                        FNEARBYINT)
+LEGACY_DAG(trunc,                            FTRUNC)
+
+#undef FUNCTION
+#undef LEGACY_DAG

llvm/include/llvm/IR/IRBuilder.h

@@ -153,8 +153,8 @@ class IRBuilderBase {
   FastMathFlags FMF;
 
   bool IsFPConstrained = false;
-  fp::ExceptionBehavior DefaultConstrainedExcept = fp::ebStrict;
-  RoundingMode DefaultConstrainedRounding = RoundingMode::Dynamic;
+  fp::ExceptionBehavior DefaultConstrainedExcept = fp::ebIgnore;
+  RoundingMode DefaultConstrainedRounding = RoundingMode::NearestTiesToEven;
 
   ArrayRef<OperandBundleDef> DefaultOperandBundles;
 
@@ -350,6 +350,19 @@ class IRBuilderBase {
   /// by this setting.
   void setIsFPConstrained(bool IsCon) { IsFPConstrained = IsCon; }
 
+  /// Enable/Disable use of constrained floating point math and reset FP options
+  /// according to the selected mode.
+  void resetModeToStrictFP(bool IsCon) {
+    if (IsCon) {
+      setDefaultConstrainedRounding(RoundingMode::Dynamic);
+      setDefaultConstrainedExcept(fp::ebStrict);
+    } else {
+      setDefaultConstrainedRounding(RoundingMode::NearestTiesToEven);
+      setDefaultConstrainedExcept(fp::ebIgnore);
+    }
+    setIsFPConstrained(IsCon);
+  }
+
   /// Query for the use of constrained floating point math
   bool getIsFPConstrained() { return IsFPConstrained; }
 
@@ -1000,6 +1013,16 @@ class IRBuilderBase {
                                      FMFSource FMFSource = {},
                                      const Twine &Name = "");
 
+  /// Create a call to intrinsic \p ID with \p Args, mangled using \p Types and
+  /// with operand bundles.
+  /// If \p FMFSource is provided, copy fast-math-flags from that instruction to
+  /// the intrinsic.
+  CallInst *CreateIntrinsic(Intrinsic::ID ID, ArrayRef<Type *> Types,
+                            ArrayRef<Value *> Args,
+                            ArrayRef<OperandBundleDef> OpBundles,
+                            Instruction *FMFSource = nullptr,
+                            const Twine &Name = "");
+
   /// Create a call to non-overloaded intrinsic \p ID with \p Args. If
   /// \p FMFSource is provided, copy fast-math-flags from that instruction to
   /// the intrinsic.
@@ -2511,24 +2534,13 @@ class IRBuilderBase {
   CallInst *CreateCall(FunctionType *FTy, Value *Callee,
                        ArrayRef<Value *> Args = {}, const Twine &Name = "",
                        MDNode *FPMathTag = nullptr) {
-    CallInst *CI = CallInst::Create(FTy, Callee, Args, DefaultOperandBundles);
-    if (IsFPConstrained)
-      setConstrainedFPCallAttr(CI);
-    if (isa<FPMathOperator>(CI))
-      setFPAttrs(CI, FPMathTag, FMF);
-    return Insert(CI, Name);
+    return CreateCall(FTy, Callee, Args, DefaultOperandBundles, Name,
+                      FPMathTag);
   }
 
   CallInst *CreateCall(FunctionType *FTy, Value *Callee, ArrayRef<Value *> Args,
                        ArrayRef<OperandBundleDef> OpBundles,
-                       const Twine &Name = "", MDNode *FPMathTag = nullptr) {
-    CallInst *CI = CallInst::Create(FTy, Callee, Args, OpBundles);
-    if (IsFPConstrained)
-      setConstrainedFPCallAttr(CI);
-    if (isa<FPMathOperator>(CI))
-      setFPAttrs(CI, FPMathTag, FMF);
-    return Insert(CI, Name);
-  }
+                       const Twine &Name = "", MDNode *FPMathTag = nullptr);
 
   CallInst *CreateCall(FunctionCallee Callee, ArrayRef<Value *> Args = {},
                        const Twine &Name = "", MDNode *FPMathTag = nullptr) {
@@ -2767,6 +2779,22 @@ class IRBuilderBase {
   /// assumption on the provided pointer.
   LLVM_ABI CallInst *CreateDereferenceableAssumption(Value *PtrValue,
                                                      Value *SizeValue);
+
+  /// Create an operand bundle in the provided bundle set to represent the given
+  /// floating-point rounding mode.
+  ///
+  /// If the rounding mode is not defined, adds the default rounding mode,
+  /// stored in this builder object.
+  void createRoundingBundle(SmallVectorImpl<OperandBundleDef> &Bundles,
+                            RoundingMode RM);
+
+  /// Create an operand bundle in the provided bundle set to represent the given
+  /// floating-point exception behavior.
+  ///
+  /// If the exception behavior is not defined, adds the default behavior,
+  /// stored in this builder object.
+  void createExceptionBundle(SmallVectorImpl<OperandBundleDef> &Bundles,
+                             fp::ExceptionBehavior Except);
 };
 
 /// This provides a uniform API for creating instructions and inserting

llvm/include/llvm/IR/InstrTypes.h

@@ -25,6 +25,7 @@
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/FMF.h"
+#include "llvm/IR/FPEnv.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/LLVMContext.h"
@@ -1093,6 +1094,18 @@ template <typename InputTy> class OperandBundleDefT {
 using OperandBundleDef = OperandBundleDefT<Value *>;
 using ConstOperandBundleDef = OperandBundleDefT<const Value *>;
 
+/// Add a bundle with tag "fp.round" and the specified rounding to the given
+/// bundle set.
+void addRoundingBundle(LLVMContext &Ctx,
+                       SmallVectorImpl<OperandBundleDef> &Bundles,
+                       RoundingMode Rounding);
+
+/// Add a bundle with tag "fp.except" and the specified exception behavior to
+/// the given bundle set.
+void addExceptionBundle(LLVMContext &Ctx,
+                        SmallVectorImpl<OperandBundleDef> &Bundles,
+                        fp::ExceptionBehavior Except);
+
 //===----------------------------------------------------------------------===//
 //                               CallBase Class
 //===----------------------------------------------------------------------===//
@@ -1152,6 +1165,9 @@ class CallBase : public Instruction {
   /// number of extra operands.
   LLVM_ABI unsigned getNumSubclassExtraOperandsDynamic() const;
 
+  /// Get memory effects specific to floating-point operations.
+  MemoryEffects getFloatingPointMemoryEffects() const;
+
 public:
   using Instruction::getContext;
 
@@ -2162,6 +2178,12 @@ class CallBase : public Instruction {
     return false;
   }
 
+  /// Return the effective rounding mode for this call.
+  RoundingMode getRoundingMode() const;
+
+  /// Return the effective exception behavior for this call.
+  fp::ExceptionBehavior getExceptionBehavior() const;
+
   /// Used to keep track of an operand bundle.  See the main comment on
   /// OperandBundleUser above.
   struct BundleOpInfo {

llvm/include/llvm/IR/IntrinsicInst.h

@@ -129,6 +129,14 @@ class IntrinsicInst : public CallInst {
   /// course of IR transformations
   LLVM_ABI static bool mayLowerToFunctionCall(Intrinsic::ID IID);
 
+  /// Check if \p ID represents a function that may access FP environment and
+  /// may have FP operand bundles.
+  ///
+  /// Access to FP environment means that in the strict FP environment the
+  /// function has read/write memory effect, which is used to maintain proper
+  /// instructions ordering.
+  static bool isFloatingPointOperation(Intrinsic::ID IID);
+
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const CallInst *I) {
     auto *F = dyn_cast_or_null<Function>(I->getCalledOperand());

llvm/include/llvm/IR/LLVMContext.h

@@ -98,7 +98,9 @@ class LLVMContext {
     OB_kcfi = 8,                   // "kcfi"
     OB_convergencectrl = 9,        // "convergencectrl"
     OB_align = 10,                 // "align"
-    OB_LastBundleID = OB_align     // Marker for last bundle ID
+    OB_fp_round = 11,              // "fp.round"
+    OB_fp_except = 12,             // "fp.except"
+    OB_LastBundleID = OB_fp_except // Marker for last bundle ID  };
   };
 
   /// getMDKindID - Return a unique non-zero ID for the specified metadata kind.

llvm/include/llvm/Support/ModRef.h

@@ -235,6 +235,11 @@ template <typename LocationEnum> class MemoryEffectsBase {
     return getWithoutLoc(Location::InaccessibleMem).doesNotAccessMemory();
   }
 
+  /// Whether this function accesses inaccessible memory.
+  bool doesAccessInaccessibleMem() const {
+    return isModOrRefSet(getModRef(Location::InaccessibleMem));
+  }
+
   /// Whether this function only (at most) accesses errno memory.
   bool onlyAccessesErrnoMem() const {
     return getWithoutLoc(Location::ErrnoMem).doesNotAccessMemory();

llvm/lib/CodeGen/AtomicExpandPass.cpp

@@ -159,7 +159,7 @@ struct ReplacementIRBuilder
     SetInsertPoint(I);
     this->CollectMetadataToCopy(I, {LLVMContext::MD_pcsections});
     if (BB->getParent()->getAttributes().hasFnAttr(Attribute::StrictFP))
-      this->setIsFPConstrained(true);
+      this->resetModeToStrictFP(true);
 
     MMRAMD = I->getMetadata(LLVMContext::MD_mmra);
   }
@@ -1716,7 +1716,7 @@ bool AtomicExpandImpl::tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
 bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
                                     CreateCmpXchgInstFun CreateCmpXchg) {
   ReplacementIRBuilder Builder(AI, AI->getDataLayout());
-  Builder.setIsFPConstrained(
+  Builder.resetModeToStrictFP(
       AI->getFunction()->hasFnAttribute(Attribute::StrictFP));
 
   // FIXME: If FP exceptions are observable, we should force them off for the