[LLVM] Change Intrinsic::ID to encode target and intrinsic index

llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h

@@ -276,7 +276,7 @@ enum {
   /// Check the operand is a specific intrinsic ID
   /// - InsnID(ULEB128) - Instruction ID
   /// - OpIdx(ULEB128) - Operand index
-  /// - IID(2) - Expected Intrinsic ID
+  /// - IID(4) - Expected Intrinsic ID
   GIM_CheckIntrinsicID,
 
   /// Check the operand is a specific predicate
@@ -412,7 +412,7 @@ enum {
 
   /// Adds an intrinsic ID to the specified instruction.
   /// - InsnID(ULEB128) - Instruction ID to modify
-  /// - IID(2) - Intrinsic ID
+  /// - IID(4) - Intrinsic ID
   GIR_AddIntrinsicID,
 
   /// Marks the implicit def of a register as dead.

llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h

@@ -889,7 +889,7 @@ bool GIMatchTableExecutor::executeMatchTable(
     case GIM_CheckIntrinsicID: {
       uint64_t InsnID = readULEB();
       uint64_t OpIdx = readULEB();
-      uint16_t Value = readU16();
+      uint32_t Value = readU32();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
                              << InsnID << "]->getOperand(" << OpIdx
@@ -1185,7 +1185,7 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
     case GIR_AddIntrinsicID: {
       uint64_t InsnID = readULEB();
-      uint16_t Value = readU16();
+      uint32_t Value = readU32();
       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
       OutMIs[InsnID].addIntrinsicID((Intrinsic::ID)Value);
       DEBUG_WITH_TYPE(TgtExecutor::getName(),

llvm/include/llvm/IR/Intrinsics.h

@@ -47,8 +47,17 @@ namespace Intrinsic {
 #define GET_INTRINSIC_ENUM_VALUES
 #include "llvm/IR/IntrinsicEnums.inc"
 #undef GET_INTRINSIC_ENUM_VALUES
+    end_id = ~0U,
   };
 
+  // Returns if `id` is a valid intrinsic ID. Validity means that it value is
+  // one of the values defined for the Intrinsic::ID enum.
+  bool IsIntrinsicIDValid(ID id);
+
+  // Get the next valid ID. This is used in test cases that iterate over valid
+  // intrinsic ID enums.
+  ID GetNextValidIntrinsicID(ID id);
+
   /// Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
   /// Note, this version is for intrinsics with no overloads.  Use the other
   /// version of getName if overloads are required.

llvm/include/llvm/Support/IntrinsicID.h

@@ -0,0 +1,53 @@
+//===- llvm/Support/IntrinsicID.h - Intrinsic ID encoding -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains functions to support intrinsic ID encoding. The
+// Intrinsic::ID enum value is constructed using a target prefix index in bits
+// 23-16 (8-bit) and an intrinsic index (index within the list of intrinsics for
+// tha target) in lower 16 bits. To support Intrinsic::ID 0 being not used, the
+// intrinsic index is encoded as Index + 1 for all targets.
+//
+// This file defines functions that encapsulate this encoding.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_INTRINSIC_ID_H
+#define LLVM_SUPPORT_INTRINSIC_ID_H
+
+#include <limits>
+#include <optional>
+#include <utility>
+
+namespace llvm::Intrinsic {
+typedef unsigned ID;
+
+inline ID encodeIntrinsicID(unsigned TargetIndex, unsigned IntrinsicIndex) {
+  assert(IntrinsicIndex < std::numeric_limits<uint16_t>::max());
+  assert(TargetIndex <= std::numeric_limits<uint8_t>::max());
+  return (TargetIndex << 16) | (IntrinsicIndex + 1);
+}
+
+inline std::pair<unsigned, unsigned> decodeIntrinsicID(ID id) {
+  unsigned IntrinsicIndex = id & 0xFFFF;
+  unsigned TargetIndex = id >> 16;
+  assert(IntrinsicIndex != 0);
+  return {TargetIndex, IntrinsicIndex - 1};
+}
+
+inline std::optional<std::pair<unsigned, unsigned>>
+decodeIntrinsicIDNoFail(ID id) {
+  unsigned IntrinsicIndex = id & 0xFFFF;
+  unsigned TargetIndex = id >> 16;
+  if (IntrinsicIndex == 0)
+    return std::nullopt;
+  return std::make_pair(TargetIndex, IntrinsicIndex - 1);
+}
+
+} // end namespace llvm::Intrinsic
+
+#endif // LLVM_SUPPORT_INTRINSIC_ID_H

llvm/lib/CodeGen/MachineOperand.cpp

@@ -997,7 +997,7 @@ void MachineOperand::print(raw_ostream &OS, ModuleSlotTracker &MST,
   }
   case MachineOperand::MO_IntrinsicID: {
     Intrinsic::ID ID = getIntrinsicID();
-    if (ID < Intrinsic::num_intrinsics)
+    if (Intrinsic::IsIntrinsicIDValid(ID))
       OS << "intrinsic(@" << Intrinsic::getBaseName(ID) << ')';
     else if (IntrinsicInfo)
       OS << "intrinsic(@" << IntrinsicInfo->getName(ID) << ')';

llvm/lib/CodeGen/MachineVerifier.cpp

@@ -1055,7 +1055,7 @@ bool MachineVerifier::verifyGIntrinsicSideEffects(const MachineInstr *MI) {
   bool NoSideEffects = Opcode == TargetOpcode::G_INTRINSIC ||
                        Opcode == TargetOpcode::G_INTRINSIC_CONVERGENT;
   unsigned IntrID = cast<GIntrinsic>(MI)->getIntrinsicID();
-  if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) {
+  if (IntrID != 0 && Intrinsic::IsIntrinsicIDValid(IntrID)) {
     AttributeList Attrs = Intrinsic::getAttributes(
         MF->getFunction().getContext(), static_cast<Intrinsic::ID>(IntrID));
     bool DeclHasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
@@ -1079,7 +1079,7 @@ bool MachineVerifier::verifyGIntrinsicConvergence(const MachineInstr *MI) {
   bool NotConvergent = Opcode == TargetOpcode::G_INTRINSIC ||
                        Opcode == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS;
   unsigned IntrID = cast<GIntrinsic>(MI)->getIntrinsicID();
-  if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) {
+  if (IntrID != 0 && Intrinsic::IsIntrinsicIDValid(IntrID)) {
     AttributeList Attrs = Intrinsic::getAttributes(
         MF->getFunction().getContext(), static_cast<Intrinsic::ID>(IntrID));
     bool DeclIsConvergent = Attrs.hasFnAttr(Attribute::Convergent);

llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp

@@ -160,7 +160,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::INTRINSIC_W_CHAIN: {
     unsigned OpNo = getOpcode() == ISD::INTRINSIC_WO_CHAIN ? 0 : 1;
     unsigned IID = getOperand(OpNo)->getAsZExtVal();
-    if (IID < Intrinsic::num_intrinsics)
+    if (Intrinsic::IsIntrinsicIDValid(IID))
       return Intrinsic::getBaseName((Intrinsic::ID)IID).str();
     if (!G)
       return "Unknown intrinsic";

llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp

@@ -4428,7 +4428,7 @@ void SelectionDAGISel::CannotYetSelect(SDNode *N) {
   } else {
     bool HasInputChain = N->getOperand(0).getValueType() == MVT::Other;
     unsigned iid = N->getConstantOperandVal(HasInputChain);
-    if (iid < Intrinsic::num_intrinsics)
+    if (Intrinsic::IsIntrinsicIDValid(iid))
       Msg << "intrinsic %" << Intrinsic::getBaseName((Intrinsic::ID)iid);
     else if (const TargetIntrinsicInfo *TII = TM.getIntrinsicInfo())
       Msg << "target intrinsic %" << TII->getName(iid);

llvm/lib/IR/Core.cpp

@@ -2458,7 +2458,8 @@ unsigned LLVMGetIntrinsicID(LLVMValueRef Fn) {
 }
 
 static Intrinsic::ID llvm_map_to_intrinsic_id(unsigned ID) {
-  assert(ID < llvm::Intrinsic::num_intrinsics && "Intrinsic ID out of range");
+  assert(llvm::Intrinsic::IsIntrinsicIDValid(ID) &&
+         "Intrinsic ID out of range");
   return llvm::Intrinsic::ID(ID);
 }
 

llvm/lib/IR/Intrinsics.cpp

@@ -30,10 +30,55 @@
 #include "llvm/IR/IntrinsicsXCore.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Type.h"
+#include "llvm/Support/IntrinsicID.h"
 
 using namespace llvm;
 using namespace Intrinsic;
 
+/// Table of per-target intrinsic name tables.
+#define GET_INTRINSIC_TARGET_DATA
+#include "llvm/IR/IntrinsicImpl.inc"
+#undef GET_INTRINSIC_TARGET_DATA
+size_t constexpr NumTargets = sizeof(TargetInfos) / sizeof(TargetInfos[0]);
+
+// Returns true if the given intrinsic ID is valid, that is, its value is one
+// of the enum values defined for this intrinsic (including not_intrinsic).
+bool Intrinsic::IsIntrinsicIDValid(ID ID) {
+  if (ID == not_intrinsic)
+    return true;
+  auto Decoded = decodeIntrinsicIDNoFail(ID);
+  if (!Decoded)
+    return false;
+  unsigned TargetIdx = Decoded->first;
+  unsigned IntrinsicIdx = Decoded->second;
+  return TargetIdx < NumTargets && IntrinsicIdx < TargetInfos[TargetIdx].Count;
+}
+
+// Returns linear index of ID if its valid, else returns 0.
+unsigned getLinearIndex(ID ID) {
+  auto Decoded = decodeIntrinsicIDNoFail(ID);
+  if (!Decoded)
+    return 0;
+  unsigned TargetIdx = Decoded->first;
+  unsigned IntrinsicIdx = Decoded->second;
+  return TargetInfos[TargetIdx].FirstLinearIndex + IntrinsicIdx;
+}
+
+ID Intrinsic::GetNextValidIntrinsicID(ID ID) {
+  if (ID == not_intrinsic)
+    return encodeIntrinsicID(0, 0);
+  if (ID == last_valid_intrinsic_id)
+    return end_id;
+  if (ID == end_id)
+    llvm_unreachable("Cannot find the next valid intrisnic");
+  auto [TargetIndex, IntrinsicIndex] = decodeIntrinsicID(ID);
+  if (IntrinsicIndex + 1 < TargetInfos[TargetIndex].Count)
+    return encodeIntrinsicID(TargetIndex, IntrinsicIndex + 1);
+  if (TargetIndex + 1 < NumTargets)
+    return encodeIntrinsicID(TargetIndex + 1, 0);
+  llvm_unreachable("Cannot find the next valid intrisnic");
+}
+
 /// Table of string intrinsic names indexed by enum value.
 static constexpr const char *const IntrinsicNameTable[] = {
     "not_intrinsic",
@@ -43,12 +88,12 @@ static constexpr const char *const IntrinsicNameTable[] = {
 };
 
 StringRef Intrinsic::getBaseName(ID id) {
-  assert(id < num_intrinsics && "Invalid intrinsic ID!");
-  return IntrinsicNameTable[id];
+  assert(IsIntrinsicIDValid(id) && "Invalid intrinsic ID!");
+  return ArrayRef(IntrinsicNameTable)[getLinearIndex(id)];
 }
 
 StringRef Intrinsic::getName(ID id) {
-  assert(id < num_intrinsics && "Invalid intrinsic ID!");
+  assert(IsIntrinsicIDValid(id) && "Invalid intrinsic ID!");
   assert(!isOverloaded(id) &&
          "This version of getName does not support overloading");
   return getBaseName(id);
@@ -155,8 +200,7 @@ static std::string getMangledTypeStr(Type *Ty, bool &HasUnnamedType) {
 static std::string getIntrinsicNameImpl(ID Id, ArrayRef<Type *> Tys, Module *M,
                                         FunctionType *FT,
                                         bool EarlyModuleCheck) {
-
-  assert(Id < num_intrinsics && "Invalid intrinsic ID!");
+  assert(IsIntrinsicIDValid(Id) && "Invalid intrinsic ID!");
   assert((Tys.empty() || isOverloaded(Id)) &&
          "This version of getName is for overloaded intrinsics only");
   (void)EarlyModuleCheck;
@@ -445,11 +489,15 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
 #undef GET_INTRINSIC_GENERATOR_GLOBAL
 
 void Intrinsic::getIntrinsicInfoTableEntries(
-    ID id, SmallVectorImpl<IITDescriptor> &T) {
+    ID IntrinsicID, SmallVectorImpl<IITDescriptor> &T) {
   static_assert(sizeof(IIT_Table[0]) == 2,
                 "Expect 16-bit entries in IIT_Table");
+  assert(IsIntrinsicIDValid(IntrinsicID));
+  unsigned Idx = getLinearIndex(IntrinsicID);
+  if (Idx == 0)
+    return;
   // Check to see if the intrinsic's type was expressible by the table.
-  uint16_t TableVal = IIT_Table[id - 1];
+  uint16_t TableVal = IIT_Table[Idx - 1];
 
   // Decode the TableVal into an array of IITValues.
   SmallVector<unsigned char> IITValues;
@@ -602,18 +650,21 @@ FunctionType *Intrinsic::getType(LLVMContext &Context, ID id,
   return FunctionType::get(ResultTy, ArgTys, false);
 }
 
-bool Intrinsic::isOverloaded(ID id) {
+// Check if an intrinsic is overloaded or not using its linear index.
+static bool isOverloadedUsingLinearIndex(unsigned Idx) {
 #define GET_INTRINSIC_OVERLOAD_TABLE
 #include "llvm/IR/IntrinsicImpl.inc"
 #undef GET_INTRINSIC_OVERLOAD_TABLE
 }
 
-/// Table of per-target intrinsic name tables.
-#define GET_INTRINSIC_TARGET_DATA
-#include "llvm/IR/IntrinsicImpl.inc"
-#undef GET_INTRINSIC_TARGET_DATA
+bool Intrinsic::isOverloaded(ID id) {
+  assert(IsIntrinsicIDValid(id));
+  return isOverloadedUsingLinearIndex(getLinearIndex(id));
+}
 
-bool Intrinsic::isTargetIntrinsic(ID IID) { return IID > TargetInfos[0].Count; }
+bool Intrinsic::isTargetIntrinsic(ID IID) {
+  return IID != not_intrinsic && decodeIntrinsicID(IID).first != 0;
+}
 
 int Intrinsic::lookupLLVMIntrinsicByName(ArrayRef<const char *> NameTable,
                                          StringRef Name, StringRef Target) {
@@ -673,7 +724,29 @@ findTargetSubtable(StringRef Name) {
   // We've either found the target or just fall back to the generic set, which
   // is always first.
   const auto &TI = It != Targets.end() && It->Name == Target ? *It : Targets[0];
-  return {ArrayRef(&IntrinsicNameTable[1] + TI.Offset, TI.Count), TI.Name};
+  unsigned LinearIndex = TI.FirstLinearIndex;
+  return {ArrayRef(IntrinsicNameTable + LinearIndex, TI.Count), TI.Name};
+}
+
+static Intrinsic::ID getIntrinsicIDFromIndex(unsigned Idx) {
+  if (Idx == 0)
+    return Intrinsic::not_intrinsic;
+
+  auto It =
+      partition_point(TargetInfos, [Idx](const IntrinsicTargetInfo &Info) {
+        return Info.FirstLinearIndex + Info.Count < Idx;
+      });
+  // Idx, if present, will be in the entry at It or It + 1.
+  if (It == std::end(TargetInfos))
+    return Intrinsic::not_intrinsic;
+  unsigned TargetIndex = std::distance(std::begin(TargetInfos), It);
+  if (It->FirstLinearIndex <= Idx && Idx < It->FirstLinearIndex + It->Count)
+    return encodeIntrinsicID(TargetIndex, Idx - It->FirstLinearIndex);
+  ++It;
+  ++TargetIndex;
+  if (It->FirstLinearIndex <= Idx && Idx < It->FirstLinearIndex + It->Count)
+    return encodeIntrinsicID(TargetIndex, Idx - It->FirstLinearIndex);
+  return Intrinsic::not_intrinsic;
 }
 
 /// This does the actual lookup of an intrinsic ID which matches the given
@@ -683,18 +756,18 @@ ID Intrinsic::lookupIntrinsicID(StringRef Name) {
   int Idx = lookupLLVMIntrinsicByName(NameTable, Name, Target);
   if (Idx == -1)
     return not_intrinsic;
-
-  // Intrinsic IDs correspond to the location in IntrinsicNameTable, but we have
-  // an index into a sub-table.
+  const auto MatchSize = strlen(NameTable[Idx]);
+  // Adjust the index from sub-table index to index into the global table.
   int Adjust = NameTable.data() - IntrinsicNameTable;
-  ID Id = static_cast<ID>(Idx + Adjust);
+  Idx += Adjust;
 
   // If the intrinsic is not overloaded, require an exact match. If it is
   // overloaded, require either exact or prefix match.
-  const auto MatchSize = strlen(NameTable[Idx]);
   assert(Name.size() >= MatchSize && "Expected either exact or prefix match");
   bool IsExactMatch = Name.size() == MatchSize;
-  return IsExactMatch || isOverloaded(Id) ? Id : not_intrinsic;
+  return IsExactMatch || isOverloadedUsingLinearIndex(Idx)
+             ? getIntrinsicIDFromIndex(Idx)
+             : not_intrinsic;
 }
 
 /// This defines the "Intrinsic::getAttributes(ID id)" method.
@@ -1028,8 +1101,9 @@ bool Intrinsic::matchIntrinsicVarArg(bool isVarArg,
 
 bool Intrinsic::getIntrinsicSignature(ID ID, FunctionType *FT,
                                       SmallVectorImpl<Type *> &ArgTys) {
-  if (!ID)
+  if (ID == Intrinsic::not_intrinsic)
     return false;
+  assert(IsIntrinsicIDValid(ID));
 
   SmallVector<IITDescriptor, 8> Table;
   getIntrinsicInfoTableEntries(ID, Table);

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

@@ -7551,7 +7551,7 @@ static unsigned getIntrinsicID(const SDNode *N) {
     return Intrinsic::not_intrinsic;
   case ISD::INTRINSIC_WO_CHAIN: {
     unsigned IID = N->getConstantOperandVal(0);
-    if (IID < Intrinsic::num_intrinsics)
+    if (Intrinsic::IsIntrinsicIDValid(IID))
       return IID;
     return Intrinsic::not_intrinsic;
   }

llvm/lib/Target/X86/X86IntrinsicsInfo.h

@@ -79,8 +79,7 @@ enum IntrinsicType : uint16_t {
 };
 
 struct IntrinsicData {
-
-  uint16_t Id;
+  unsigned Id;
   IntrinsicType Type;
   uint16_t Opc0;
   uint16_t Opc1;