[CodeGen] Virtualize isTargetStrictFPOpcode / isTargetMemoryOpcode

Author: s-barannikov

llvm/include/llvm/CodeGen/ISDOpcodes.h

@@ -1490,17 +1490,6 @@ enum NodeType {
   BUILTIN_OP_END
 };
 
-/// FIRST_TARGET_STRICTFP_OPCODE - Target-specific pre-isel operations
-/// which cannot raise FP exceptions should be less than this value.
-/// Those that do must not be less than this value.
-static const int FIRST_TARGET_STRICTFP_OPCODE = BUILTIN_OP_END + 400;
-
-/// FIRST_TARGET_MEMORY_OPCODE - Target-specific pre-isel operations
-/// which do not reference a specific memory location should be less than
-/// this value. Those that do must not be less than this value, and can
-/// be used with SelectionDAG::getMemIntrinsicNode.
-static const int FIRST_TARGET_MEMORY_OPCODE = BUILTIN_OP_END + 500;
-
 /// Whether this is bitwise logic opcode.
 inline bool isBitwiseLogicOp(unsigned Opcode) {
   return Opcode == ISD::AND || Opcode == ISD::OR || Opcode == ISD::XOR;

llvm/include/llvm/CodeGen/SelectionDAG.h

@@ -1327,8 +1327,8 @@ class SelectionDAG {
 
   /// Creates a MemIntrinsicNode that may produce a
   /// result and takes a list of operands. Opcode may be INTRINSIC_VOID,
-  /// INTRINSIC_W_CHAIN, or a target-specific opcode with a value not
-  /// less than FIRST_TARGET_MEMORY_OPCODE.
+  /// INTRINSIC_W_CHAIN, or a target-specific memory-referencing opcode
+  // (see `SelectionDAGTargetInfo::isTargetMemoryOpcode`).
   SDValue getMemIntrinsicNode(
       unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
       EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,

llvm/include/llvm/CodeGen/SelectionDAGNodes.h

@@ -210,7 +210,6 @@ class SDValue {
   inline const SDValue &getOperand(unsigned i) const;
   inline uint64_t getConstantOperandVal(unsigned i) const;
   inline const APInt &getConstantOperandAPInt(unsigned i) const;
-  inline bool isTargetMemoryOpcode() const;
   inline bool isTargetOpcode() const;
   inline bool isMachineOpcode() const;
   inline bool isUndef() const;
@@ -688,22 +687,6 @@ END_TWO_BYTE_PACK()
   /// \<target\>ISD namespace).
   bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
 
-  /// Test if this node has a target-specific opcode that may raise
-  /// FP exceptions (in the \<target\>ISD namespace and greater than
-  /// FIRST_TARGET_STRICTFP_OPCODE).  Note that all target memory
-  /// opcode are currently automatically considered to possibly raise
-  /// FP exceptions as well.
-  bool isTargetStrictFPOpcode() const {
-    return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
-  }
-
-  /// Test if this node has a target-specific
-  /// memory-referencing opcode (in the \<target\>ISD namespace and
-  /// greater than FIRST_TARGET_MEMORY_OPCODE).
-  bool isTargetMemoryOpcode() const {
-    return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
-  }
-
   /// Return true if the type of the node type undefined.
   bool isUndef() const { return NodeType == ISD::UNDEF; }
 
@@ -1214,10 +1197,6 @@ inline bool SDValue::isTargetOpcode() const {
   return Node->isTargetOpcode();
 }
 
-inline bool SDValue::isTargetMemoryOpcode() const {
-  return Node->isTargetMemoryOpcode();
-}
-
 inline bool SDValue::isMachineOpcode() const {
   return Node->isMachineOpcode();
 }
@@ -1571,10 +1550,10 @@ class AtomicSDNode : public MemSDNode {
   }
 };
 
-/// This SDNode is used for target intrinsics that touch
-/// memory and need an associated MachineMemOperand. Its opcode may be
-/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
-/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
+/// This SDNode is used for target intrinsics that touch memory and need
+/// an associated MachineMemOperand. Its opcode may be INTRINSIC_VOID,
+/// INTRINSIC_W_CHAIN, PREFETCH, or a target-specific memory-referencing
+/// opcode (see `SelectionDAGTargetInfo::isTargetMemoryOpcode`).
 class MemIntrinsicSDNode : public MemSDNode {
 public:
   MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,

llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h

@@ -35,6 +35,19 @@ class SelectionDAGTargetInfo {
   SelectionDAGTargetInfo &operator=(const SelectionDAGTargetInfo &) = delete;
   virtual ~SelectionDAGTargetInfo();
 
+  /// Returns true if a node with the given target-specific opcode has
+  /// a memory operand. Nodes with such opcodes can only be created with
+  /// `SelectionDAG::getMemIntrinsicNode`.
+  virtual bool isTargetMemoryOpcode(unsigned Opcode) const { return false; }
+
+  /// Returns true if a node with the given target-specific opcode has
+  /// strict floating-point semantics.
+  virtual bool isTargetStrictFPOpcode(unsigned Opcode) const { return false; }
+
+  /// Returns true if a node with the given target-specific opcode
+  /// may raise a floating-point exception.
+  virtual bool mayRaiseFPException(unsigned Opcode) const;
+
   /// Emit target-specific code that performs a memcpy.
   /// This can be used by targets to provide code sequences for cases
   /// that don't fit the target's parameters for simple loads/stores and can be

llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

@@ -8978,12 +8978,12 @@ SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
                                           SDVTList VTList,
                                           ArrayRef<SDValue> Ops, EVT MemVT,
                                           MachineMemOperand *MMO) {
-  assert((Opcode == ISD::INTRINSIC_VOID ||
-          Opcode == ISD::INTRINSIC_W_CHAIN ||
-          Opcode == ISD::PREFETCH ||
-          (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
-           (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
-         "Opcode is not a memory-accessing opcode!");
+  assert(
+      (Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN ||
+       Opcode == ISD::PREFETCH ||
+       (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
+        Opcode >= ISD::BUILTIN_OP_END && TSI->isTargetMemoryOpcode(Opcode))) &&
+      "Opcode is not a memory-accessing opcode!");
 
   // Memoize the node unless it returns a glue result.
   MemIntrinsicSDNode *N;

llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp

@@ -51,6 +51,7 @@
 #include "llvm/CodeGen/SchedulerRegistry.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
+#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
 #include "llvm/CodeGen/StackMaps.h"
 #include "llvm/CodeGen/StackProtector.h"
 #include "llvm/CodeGen/SwiftErrorValueTracking.h"
@@ -4394,8 +4395,10 @@ bool SelectionDAGISel::mayRaiseFPException(SDNode *N) const {
 
   // For ISD opcodes, only StrictFP opcodes may raise an FP
   // exception.
-  if (N->isTargetOpcode())
-    return N->isTargetStrictFPOpcode();
+  if (N->isTargetOpcode()) {
+    const SelectionDAGTargetInfo &TSI = CurDAG->getSelectionDAGInfo();
+    return TSI.mayRaiseFPException(N->getOpcode());
+  }
   return N->isStrictFPOpcode();
 }
 

llvm/lib/CodeGen/SelectionDAG/SelectionDAGTargetInfo.cpp

@@ -15,3 +15,9 @@
 using namespace llvm;
 
 SelectionDAGTargetInfo::~SelectionDAGTargetInfo() = default;
+
+bool SelectionDAGTargetInfo::mayRaiseFPException(unsigned Opcode) const {
+  // FIXME: All target memory opcodes are currently automatically considered
+  //  to possibly raise FP exceptions. See rev. 63336795.
+  return isTargetStrictFPOpcode(Opcode) || isTargetMemoryOpcode(Opcode);
+}

llvm/lib/Target/AArch64/AArch64ISelLowering.h

@@ -477,15 +477,16 @@ enum NodeType : unsigned {
   MSRR,
 
   // Strict (exception-raising) floating point comparison
-  STRICT_FCMP = ISD::FIRST_TARGET_STRICTFP_OPCODE,
+  STRICT_FCMP,
   STRICT_FCMPE,
 
   // SME ZA loads and stores
   SME_ZA_LDR,
   SME_ZA_STR,
 
   // NEON Load/Store with post-increment base updates
-  LD2post = ISD::FIRST_TARGET_MEMORY_OPCODE,
+  FIRST_MEMORY_OPCODE,
+  LD2post = FIRST_MEMORY_OPCODE,
   LD3post,
   LD4post,
   ST2post,
@@ -520,6 +521,7 @@ enum NodeType : unsigned {
   STP,
   STILP,
   STNP,
+  LAST_MEMORY_OPCODE = STNP,
 };
 
 } // end namespace AArch64ISD

llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp

@@ -23,6 +23,26 @@ static cl::opt<bool>
                                 "to lower to librt functions"),
                        cl::init(true));
 
+bool AArch64SelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const {
+  if (Opcode >= AArch64ISD::FIRST_MEMORY_OPCODE &&
+      Opcode <= AArch64ISD::LAST_MEMORY_OPCODE)
+    return true;
+  return SelectionDAGTargetInfo::isTargetMemoryOpcode(Opcode);
+}
+
+bool AArch64SelectionDAGInfo::isTargetStrictFPOpcode(unsigned Opcode) const {
+  switch (static_cast<AArch64ISD::NodeType>(Opcode)) {
+  default:
+    break;
+  case AArch64ISD::STRICT_FCMP:
+  case AArch64ISD::STRICT_FCMPE:
+  case AArch64ISD::SME_ZA_LDR:
+  case AArch64ISD::SME_ZA_STR:
+    return true;
+  }
+  return SelectionDAGTargetInfo::isTargetStrictFPOpcode(Opcode);
+}
+
 SDValue AArch64SelectionDAGInfo::EmitMOPS(unsigned Opcode, SelectionDAG &DAG,
                                           const SDLoc &DL, SDValue Chain,
                                           SDValue Dst, SDValue SrcOrValue,

llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.h

@@ -19,6 +19,10 @@ namespace llvm {
 
 class AArch64SelectionDAGInfo : public SelectionDAGTargetInfo {
 public:
+  bool isTargetMemoryOpcode(unsigned Opcode) const override;
+
+  bool isTargetStrictFPOpcode(unsigned Opcode) const override;
+
   SDValue EmitMOPS(unsigned Opcode, SelectionDAG &DAG, const SDLoc &DL,
                    SDValue Chain, SDValue Dst, SDValue SrcOrValue, SDValue Size,
                    Align Alignment, bool isVolatile,