[SelectionDAG] Virtualize isTargetStrictFPOpcode / isTargetMemoryOpcode

llvm/include/llvm/CodeGen/ISDOpcodes.h

@@ -1490,17 +1490,6 @@ enum NodeType {
   BUILTIN_OP_END
 };
 
-/// FIRST_TARGET_STRICTFP_OPCODE - Target-specific pre-isel operations
-/// which cannot raise FP exceptions should be less than this value.
-/// Those that do must not be less than this value.
-static const int FIRST_TARGET_STRICTFP_OPCODE = BUILTIN_OP_END + 400;
-
-/// FIRST_TARGET_MEMORY_OPCODE - Target-specific pre-isel operations
-/// which do not reference a specific memory location should be less than
-/// this value. Those that do must not be less than this value, and can
-/// be used with SelectionDAG::getMemIntrinsicNode.
-static const int FIRST_TARGET_MEMORY_OPCODE = BUILTIN_OP_END + 500;
-
 /// Whether this is bitwise logic opcode.
 inline bool isBitwiseLogicOp(unsigned Opcode) {
   return Opcode == ISD::AND || Opcode == ISD::OR || Opcode == ISD::XOR;

llvm/include/llvm/CodeGen/SelectionDAG.h

@@ -1327,8 +1327,8 @@ class SelectionDAG {
 
   /// Creates a MemIntrinsicNode that may produce a
   /// result and takes a list of operands. Opcode may be INTRINSIC_VOID,
-  /// INTRINSIC_W_CHAIN, or a target-specific opcode with a value not
-  /// less than FIRST_TARGET_MEMORY_OPCODE.
+  /// INTRINSIC_W_CHAIN, or a target-specific memory-referencing opcode
+  // (see `SelectionDAGTargetInfo::isTargetMemoryOpcode`).
   SDValue getMemIntrinsicNode(
       unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
       EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,

llvm/include/llvm/CodeGen/SelectionDAGNodes.h

@@ -210,7 +210,6 @@ class SDValue {
   inline const SDValue &getOperand(unsigned i) const;
   inline uint64_t getConstantOperandVal(unsigned i) const;
   inline const APInt &getConstantOperandAPInt(unsigned i) const;
-  inline bool isTargetMemoryOpcode() const;
   inline bool isTargetOpcode() const;
   inline bool isMachineOpcode() const;
   inline bool isUndef() const;
@@ -688,22 +687,6 @@ END_TWO_BYTE_PACK()
   /// \<target\>ISD namespace).
   bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
 
-  /// Test if this node has a target-specific opcode that may raise
-  /// FP exceptions (in the \<target\>ISD namespace and greater than
-  /// FIRST_TARGET_STRICTFP_OPCODE).  Note that all target memory
-  /// opcode are currently automatically considered to possibly raise
-  /// FP exceptions as well.
-  bool isTargetStrictFPOpcode() const {
-    return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
-  }
-
-  /// Test if this node has a target-specific
-  /// memory-referencing opcode (in the \<target\>ISD namespace and
-  /// greater than FIRST_TARGET_MEMORY_OPCODE).
-  bool isTargetMemoryOpcode() const {
-    return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
-  }
-
   /// Return true if the type of the node type undefined.
   bool isUndef() const { return NodeType == ISD::UNDEF; }
 
@@ -1214,10 +1197,6 @@ inline bool SDValue::isTargetOpcode() const {
   return Node->isTargetOpcode();
 }
 
-inline bool SDValue::isTargetMemoryOpcode() const {
-  return Node->isTargetMemoryOpcode();
-}
-
 inline bool SDValue::isMachineOpcode() const {
   return Node->isMachineOpcode();
 }
@@ -1571,10 +1550,10 @@ class AtomicSDNode : public MemSDNode {
   }
 };
 
-/// This SDNode is used for target intrinsics that touch
-/// memory and need an associated MachineMemOperand. Its opcode may be
-/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
-/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
+/// This SDNode is used for target intrinsics that touch memory and need
+/// an associated MachineMemOperand. Its opcode may be INTRINSIC_VOID,
+/// INTRINSIC_W_CHAIN, PREFETCH, or a target-specific memory-referencing
+/// opcode (see `SelectionDAGTargetInfo::isTargetMemoryOpcode`).
 class MemIntrinsicSDNode : public MemSDNode {
 public:
   MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,

llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h

@@ -35,6 +35,19 @@ class SelectionDAGTargetInfo {
   SelectionDAGTargetInfo &operator=(const SelectionDAGTargetInfo &) = delete;
   virtual ~SelectionDAGTargetInfo();
 
+  /// Returns true if a node with the given target-specific opcode has
+  /// a memory operand. Nodes with such opcodes can only be created with
+  /// `SelectionDAG::getMemIntrinsicNode`.
+  virtual bool isTargetMemoryOpcode(unsigned Opcode) const { return false; }
+
+  /// Returns true if a node with the given target-specific opcode has
+  /// strict floating-point semantics.
+  virtual bool isTargetStrictFPOpcode(unsigned Opcode) const { return false; }
+
+  /// Returns true if a node with the given target-specific opcode
+  /// may raise a floating-point exception.
+  virtual bool mayRaiseFPException(unsigned Opcode) const;
+
   /// Emit target-specific code that performs a memcpy.
   /// This can be used by targets to provide code sequences for cases
   /// that don't fit the target's parameters for simple loads/stores and can be

llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

@@ -8978,12 +8978,12 @@ SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
                                           SDVTList VTList,
                                           ArrayRef<SDValue> Ops, EVT MemVT,
                                           MachineMemOperand *MMO) {
-  assert((Opcode == ISD::INTRINSIC_VOID ||
-          Opcode == ISD::INTRINSIC_W_CHAIN ||
-          Opcode == ISD::PREFETCH ||
-          (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
-           (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
-         "Opcode is not a memory-accessing opcode!");
+  assert(
+      (Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN ||
+       Opcode == ISD::PREFETCH ||
+       (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
+        Opcode >= ISD::BUILTIN_OP_END && TSI->isTargetMemoryOpcode(Opcode))) &&
+      "Opcode is not a memory-accessing opcode!");
 
   // Memoize the node unless it returns a glue result.
   MemIntrinsicSDNode *N;

llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp

@@ -51,6 +51,7 @@
 #include "llvm/CodeGen/SchedulerRegistry.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
+#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
 #include "llvm/CodeGen/StackMaps.h"
 #include "llvm/CodeGen/StackProtector.h"
 #include "llvm/CodeGen/SwiftErrorValueTracking.h"
@@ -4395,8 +4396,10 @@ bool SelectionDAGISel::mayRaiseFPException(SDNode *N) const {
 
   // For ISD opcodes, only StrictFP opcodes may raise an FP
   // exception.
-  if (N->isTargetOpcode())
-    return N->isTargetStrictFPOpcode();
+  if (N->isTargetOpcode()) {
+    const SelectionDAGTargetInfo &TSI = CurDAG->getSelectionDAGInfo();
+    return TSI.mayRaiseFPException(N->getOpcode());
+  }
   return N->isStrictFPOpcode();
 }
 

llvm/lib/CodeGen/SelectionDAG/SelectionDAGTargetInfo.cpp

@@ -15,3 +15,9 @@
 using namespace llvm;
 
 SelectionDAGTargetInfo::~SelectionDAGTargetInfo() = default;
+
+bool SelectionDAGTargetInfo::mayRaiseFPException(unsigned Opcode) const {
+  // FIXME: All target memory opcodes are currently automatically considered
+  //  to possibly raise FP exceptions. See rev. 63336795.
+  return isTargetStrictFPOpcode(Opcode) || isTargetMemoryOpcode(Opcode);
+}

llvm/lib/Target/AArch64/AArch64ISelLowering.h

@@ -477,15 +477,16 @@ enum NodeType : unsigned {
   MSRR,
 
   // Strict (exception-raising) floating point comparison
-  STRICT_FCMP = ISD::FIRST_TARGET_STRICTFP_OPCODE,
+  STRICT_FCMP,
   STRICT_FCMPE,
 
   // SME ZA loads and stores
   SME_ZA_LDR,
   SME_ZA_STR,
 
   // NEON Load/Store with post-increment base updates
-  LD2post = ISD::FIRST_TARGET_MEMORY_OPCODE,
+  FIRST_MEMORY_OPCODE,
+  LD2post = FIRST_MEMORY_OPCODE,
   LD3post,
   LD4post,
   ST2post,
@@ -520,6 +521,7 @@ enum NodeType : unsigned {
   STP,
   STILP,
   STNP,
+  LAST_MEMORY_OPCODE = STNP,
 };
 
 } // end namespace AArch64ISD

llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp

@@ -23,6 +23,23 @@ static cl::opt<bool>
                                 "to lower to librt functions"),
                        cl::init(true));
 
+bool AArch64SelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const {
+  return Opcode >= AArch64ISD::FIRST_MEMORY_OPCODE &&
+         Opcode <= AArch64ISD::LAST_MEMORY_OPCODE;
+}
+
+bool AArch64SelectionDAGInfo::isTargetStrictFPOpcode(unsigned Opcode) const {
+  switch (static_cast<AArch64ISD::NodeType>(Opcode)) {
+  default:
+    return false;
+  case AArch64ISD::STRICT_FCMP:
+  case AArch64ISD::STRICT_FCMPE:
+  case AArch64ISD::SME_ZA_LDR:
+  case AArch64ISD::SME_ZA_STR:
+    return true;
+  }
+}
+
 SDValue AArch64SelectionDAGInfo::EmitMOPS(unsigned Opcode, SelectionDAG &DAG,
                                           const SDLoc &DL, SDValue Chain,
                                           SDValue Dst, SDValue SrcOrValue,

llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.h

@@ -19,6 +19,10 @@ namespace llvm {
 
 class AArch64SelectionDAGInfo : public SelectionDAGTargetInfo {
 public:
+  bool isTargetMemoryOpcode(unsigned Opcode) const override;
+
+  bool isTargetStrictFPOpcode(unsigned Opcode) const override;
+
   SDValue EmitMOPS(unsigned Opcode, SelectionDAG &DAG, const SDLoc &DL,
                    SDValue Chain, SDValue Dst, SDValue SrcOrValue, SDValue Size,
                    Align Alignment, bool isVolatile,

llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp

@@ -5555,7 +5555,6 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(PC_ADD_REL_OFFSET)
   NODE_NAME_CASE(LDS)
   NODE_NAME_CASE(DUMMY_CHAIN)
-  case AMDGPUISD::FIRST_MEM_OPCODE_NUMBER: break;
   NODE_NAME_CASE(LOAD_D16_HI)
   NODE_NAME_CASE(LOAD_D16_LO)
   NODE_NAME_CASE(LOAD_D16_HI_I8)

llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h

@@ -546,8 +546,9 @@ enum NodeType : unsigned {
   LDS,
 
   DUMMY_CHAIN,
-  FIRST_MEM_OPCODE_NUMBER = ISD::FIRST_TARGET_MEMORY_OPCODE,
-  LOAD_D16_HI,
+
+  FIRST_MEMORY_OPCODE,
+  LOAD_D16_HI = FIRST_MEMORY_OPCODE,
   LOAD_D16_LO,
   LOAD_D16_HI_I8,
   LOAD_D16_HI_U8,
@@ -603,6 +604,7 @@ enum NodeType : unsigned {
   BUFFER_ATOMIC_FMIN,
   BUFFER_ATOMIC_FMAX,
   BUFFER_ATOMIC_COND_SUB_U32,
+  LAST_MEMORY_OPCODE = BUFFER_ATOMIC_COND_SUB_U32,
 };
 
 } // End namespace AMDGPUISD

llvm/lib/Target/AMDGPU/AMDGPUSelectionDAGInfo.cpp

@@ -7,7 +7,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPUSelectionDAGInfo.h"
+#include "AMDGPUISelLowering.h"
 
 using namespace llvm;
 
 AMDGPUSelectionDAGInfo::~AMDGPUSelectionDAGInfo() = default;
+
+bool AMDGPUSelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const {
+  return Opcode >= AMDGPUISD::FIRST_MEMORY_OPCODE &&
+         Opcode <= AMDGPUISD::LAST_MEMORY_OPCODE;
+}

llvm/lib/Target/AMDGPU/AMDGPUSelectionDAGInfo.h

@@ -16,6 +16,8 @@ namespace llvm {
 class AMDGPUSelectionDAGInfo : public SelectionDAGTargetInfo {
 public:
   ~AMDGPUSelectionDAGInfo() override;
+
+  bool isTargetMemoryOpcode(unsigned Opcode) const override;
 };
 
 } // namespace llvm

llvm/lib/Target/ARM/ARMISelLowering.h

@@ -321,7 +321,8 @@ class VectorType;
     CSINC, // Conditional select increment.
 
     // Vector load N-element structure to all lanes:
-    VLD1DUP = ISD::FIRST_TARGET_MEMORY_OPCODE,
+    FIRST_MEMORY_OPCODE,
+    VLD1DUP = FIRST_MEMORY_OPCODE,
     VLD2DUP,
     VLD3DUP,
     VLD4DUP,
@@ -356,7 +357,8 @@ class VectorType;
 
     // Load/Store of dual registers
     LDRD,
-    STRD
+    STRD,
+    LAST_MEMORY_OPCODE = STRD,
   };
 
   } // end namespace ARMISD

llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp

@@ -30,6 +30,11 @@ cl::opt<TPLoop::MemTransfer> EnableMemtransferTPLoop(
                           "Allow (may be subject to certain conditions) "
                           "conversion of memcpy to TP loop.")));
 
+bool ARMSelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const {
+  return Opcode >= ARMISD::FIRST_MEMORY_OPCODE &&
+         Opcode <= ARMISD::LAST_MEMORY_OPCODE;
+}
+
 // Emit, if possible, a specialized version of the given Libcall. Typically this
 // means selecting the appropriately aligned version, but we also convert memset
 // of 0 into memclr.

llvm/lib/Target/ARM/ARMSelectionDAGInfo.h

@@ -37,6 +37,8 @@ namespace ARM_AM {
 
 class ARMSelectionDAGInfo : public SelectionDAGTargetInfo {
 public:
+  bool isTargetMemoryOpcode(unsigned Opcode) const override;
+
   SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
                                   SDValue Chain, SDValue Dst, SDValue Src,
                                   SDValue Size, Align Alignment,

llvm/lib/Target/Mips/MipsISelLowering.h

@@ -247,7 +247,7 @@ class TargetRegisterClass;
       DOUBLE_SELECT_I64,
 
       // Load/Store Left/Right nodes.
-      LWL = ISD::FIRST_TARGET_MEMORY_OPCODE,
+      LWL,
       LWR,
       SWL,
       SWR,

llvm/lib/Target/Mips/MipsSelectionDAGInfo.cpp

@@ -7,7 +7,24 @@
 //===----------------------------------------------------------------------===//
 
 #include "MipsSelectionDAGInfo.h"
+#include "MipsISelLowering.h"
 
 using namespace llvm;
 
 MipsSelectionDAGInfo::~MipsSelectionDAGInfo() = default;
+
+bool MipsSelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const {
+  switch (static_cast<MipsISD::NodeType>(Opcode)) {
+  default:
+    return false;
+  case MipsISD::LWL:
+  case MipsISD::LWR:
+  case MipsISD::SWL:
+  case MipsISD::SWR:
+  case MipsISD::LDL:
+  case MipsISD::LDR:
+  case MipsISD::SDL:
+  case MipsISD::SDR:
+    return true;
+  }
+}

llvm/lib/Target/Mips/MipsSelectionDAGInfo.h

@@ -16,6 +16,8 @@ namespace llvm {
 class MipsSelectionDAGInfo : public SelectionDAGTargetInfo {
 public:
   ~MipsSelectionDAGInfo() override;
+
+  bool isTargetMemoryOpcode(unsigned Opcode) const override;
 };
 
 } // namespace llvm

llvm/lib/Target/NVPTX/NVPTXISelLowering.h

@@ -70,7 +70,8 @@ enum NodeType : unsigned {
   BrxEnd,
   Dummy,
 
-  LoadV2 = ISD::FIRST_TARGET_MEMORY_OPCODE,
+  FIRST_MEMORY_OPCODE,
+  LoadV2 = FIRST_MEMORY_OPCODE,
   LoadV4,
   LDUV2, // LDU.v2
   LDUV4, // LDU.v4
@@ -443,7 +444,8 @@ enum NodeType : unsigned {
   Suld3DV2I64Zero,
   Suld3DV4I8Zero,
   Suld3DV4I16Zero,
-  Suld3DV4I32Zero
+  Suld3DV4I32Zero,
+  LAST_MEMORY_OPCODE = Suld3DV4I32Zero,
 };
 }
 

llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.cpp

@@ -7,7 +7,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "NVPTXSelectionDAGInfo.h"
+#include "NVPTXISelLowering.h"
 
 using namespace llvm;
 
 NVPTXSelectionDAGInfo::~NVPTXSelectionDAGInfo() = default;
+
+bool NVPTXSelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const {
+  return Opcode >= NVPTXISD::FIRST_MEMORY_OPCODE &&
+         Opcode <= NVPTXISD::LAST_MEMORY_OPCODE;
+}

llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.h

@@ -16,6 +16,8 @@ namespace llvm {
 class NVPTXSelectionDAGInfo : public SelectionDAGTargetInfo {
 public:
   ~NVPTXSelectionDAGInfo() override;
+
+  bool isTargetMemoryOpcode(unsigned Opcode) const override;
 };
 
 } // namespace llvm