diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h index b4ff4cd178d75..ec5c55074add9 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h @@ -276,7 +276,7 @@ enum { /// Check the operand is a specific intrinsic ID /// - InsnID(ULEB128) - Instruction ID /// - OpIdx(ULEB128) - Operand index - /// - IID(2) - Expected Intrinsic ID + /// - IID(4) - Expected Intrinsic ID GIM_CheckIntrinsicID, /// Check the operand is a specific predicate @@ -412,7 +412,7 @@ enum { /// Adds an intrinsic ID to the specified instruction. /// - InsnID(ULEB128) - Instruction ID to modify - /// - IID(2) - Intrinsic ID + /// - IID(4) - Intrinsic ID GIR_AddIntrinsicID, /// Marks the implicit def of a register as dead. diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h index 9f8eb030a96c6..775998fe53a7b 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h @@ -889,7 +889,7 @@ bool GIMatchTableExecutor::executeMatchTable( case GIM_CheckIntrinsicID: { uint64_t InsnID = readULEB(); uint64_t OpIdx = readULEB(); - uint16_t Value = readU16(); + uint32_t Value = readU32(); DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs[" << InsnID << "]->getOperand(" << OpIdx @@ -1185,7 +1185,7 @@ bool GIMatchTableExecutor::executeMatchTable( } case GIR_AddIntrinsicID: { uint64_t InsnID = readULEB(); - uint16_t Value = readU16(); + uint32_t Value = readU32(); assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); OutMIs[InsnID].addIntrinsicID((Intrinsic::ID)Value); DEBUG_WITH_TYPE(TgtExecutor::getName(), diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h index 89dfff256e0c4..e12c28b1a4f19 100644 --- a/llvm/include/llvm/IR/Intrinsics.h +++ b/llvm/include/llvm/IR/Intrinsics.h @@ -47,8 +47,17 @@ namespace Intrinsic { #define GET_INTRINSIC_ENUM_VALUES #include "llvm/IR/IntrinsicEnums.inc" #undef GET_INTRINSIC_ENUM_VALUES + end_id = ~0U, }; + // Returns if `id` is a valid intrinsic ID. Validity means that it value is + // one of the values defined for the Intrinsic::ID enum. + bool IsIntrinsicIDValid(ID id); + + // Get the next valid ID. This is used in test cases that iterate over valid + // intrinsic ID enums. + ID GetNextValidIntrinsicID(ID id); + /// Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx". /// Note, this version is for intrinsics with no overloads. Use the other /// version of getName if overloads are required. diff --git a/llvm/include/llvm/Support/IntrinsicID.h b/llvm/include/llvm/Support/IntrinsicID.h new file mode 100644 index 0000000000000..c1e546c6d4f35 --- /dev/null +++ b/llvm/include/llvm/Support/IntrinsicID.h @@ -0,0 +1,53 @@ +//===- llvm/Support/IntrinsicID.h - Intrinsic ID encoding -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains functions to support intrinsic ID encoding. The +// Intrinsic::ID enum value is constructed using a target prefix index in bits +// 23-16 (8-bit) and an intrinsic index (index within the list of intrinsics for +// tha target) in lower 16 bits. To support Intrinsic::ID 0 being not used, the +// intrinsic index is encoded as Index + 1 for all targets. +// +// This file defines functions that encapsulate this encoding. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_INTRINSIC_ID_H +#define LLVM_SUPPORT_INTRINSIC_ID_H + +#include +#include +#include + +namespace llvm::Intrinsic { +typedef unsigned ID; + +inline ID encodeIntrinsicID(unsigned TargetIndex, unsigned IntrinsicIndex) { + assert(IntrinsicIndex < std::numeric_limits::max()); + assert(TargetIndex <= std::numeric_limits::max()); + return (TargetIndex << 16) | (IntrinsicIndex + 1); +} + +inline std::pair decodeIntrinsicID(ID id) { + unsigned IntrinsicIndex = id & 0xFFFF; + unsigned TargetIndex = id >> 16; + assert(IntrinsicIndex != 0); + return {TargetIndex, IntrinsicIndex - 1}; +} + +inline std::optional> +decodeIntrinsicIDNoFail(ID id) { + unsigned IntrinsicIndex = id & 0xFFFF; + unsigned TargetIndex = id >> 16; + if (IntrinsicIndex == 0) + return std::nullopt; + return std::make_pair(TargetIndex, IntrinsicIndex - 1); +} + +} // end namespace llvm::Intrinsic + +#endif // LLVM_SUPPORT_INTRINSIC_ID_H diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp index d9e5e9d9d1e41..f145b5b775944 100644 --- a/llvm/lib/CodeGen/MachineOperand.cpp +++ b/llvm/lib/CodeGen/MachineOperand.cpp @@ -997,7 +997,7 @@ void MachineOperand::print(raw_ostream &OS, ModuleSlotTracker &MST, } case MachineOperand::MO_IntrinsicID: { Intrinsic::ID ID = getIntrinsicID(); - if (ID < Intrinsic::num_intrinsics) + if (Intrinsic::IsIntrinsicIDValid(ID)) OS << "intrinsic(@" << Intrinsic::getBaseName(ID) << ')'; else if (IntrinsicInfo) OS << "intrinsic(@" << IntrinsicInfo->getName(ID) << ')'; diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index e2c09fe25d55c..77440e5e7275f 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -1055,7 +1055,7 @@ bool MachineVerifier::verifyGIntrinsicSideEffects(const MachineInstr *MI) { bool NoSideEffects = Opcode == TargetOpcode::G_INTRINSIC || Opcode == TargetOpcode::G_INTRINSIC_CONVERGENT; unsigned IntrID = cast(MI)->getIntrinsicID(); - if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) { + if (IntrID != 0 && Intrinsic::IsIntrinsicIDValid(IntrID)) { AttributeList Attrs = Intrinsic::getAttributes( MF->getFunction().getContext(), static_cast(IntrID)); bool DeclHasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory(); @@ -1079,7 +1079,7 @@ bool MachineVerifier::verifyGIntrinsicConvergence(const MachineInstr *MI) { bool NotConvergent = Opcode == TargetOpcode::G_INTRINSIC || Opcode == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS; unsigned IntrID = cast(MI)->getIntrinsicID(); - if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) { + if (IntrID != 0 && Intrinsic::IsIntrinsicIDValid(IntrID)) { AttributeList Attrs = Intrinsic::getAttributes( MF->getFunction().getContext(), static_cast(IntrID)); bool DeclIsConvergent = Attrs.hasFnAttr(Attribute::Convergent); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp index 580ff19065557..fe1f53a17a77d 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -160,7 +160,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::INTRINSIC_W_CHAIN: { unsigned OpNo = getOpcode() == ISD::INTRINSIC_WO_CHAIN ? 0 : 1; unsigned IID = getOperand(OpNo)->getAsZExtVal(); - if (IID < Intrinsic::num_intrinsics) + if (Intrinsic::IsIntrinsicIDValid(IID)) return Intrinsic::getBaseName((Intrinsic::ID)IID).str(); if (!G) return "Unknown intrinsic"; diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 0d99ae9cdebd5..46712091ecd51 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -4428,7 +4428,7 @@ void SelectionDAGISel::CannotYetSelect(SDNode *N) { } else { bool HasInputChain = N->getOperand(0).getValueType() == MVT::Other; unsigned iid = N->getConstantOperandVal(HasInputChain); - if (iid < Intrinsic::num_intrinsics) + if (Intrinsic::IsIntrinsicIDValid(iid)) Msg << "intrinsic %" << Intrinsic::getBaseName((Intrinsic::ID)iid); else if (const TargetIntrinsicInfo *TII = TM.getIntrinsicInfo()) Msg << "target intrinsic %" << TII->getName(iid); diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp index dc5ca68bd9985..0ed84918dc872 100644 --- a/llvm/lib/IR/Core.cpp +++ b/llvm/lib/IR/Core.cpp @@ -2458,7 +2458,8 @@ unsigned LLVMGetIntrinsicID(LLVMValueRef Fn) { } static Intrinsic::ID llvm_map_to_intrinsic_id(unsigned ID) { - assert(ID < llvm::Intrinsic::num_intrinsics && "Intrinsic ID out of range"); + assert(llvm::Intrinsic::IsIntrinsicIDValid(ID) && + "Intrinsic ID out of range"); return llvm::Intrinsic::ID(ID); } diff --git a/llvm/lib/IR/Intrinsics.cpp b/llvm/lib/IR/Intrinsics.cpp index 459b4d2f62276..29b425aa81a88 100644 --- a/llvm/lib/IR/Intrinsics.cpp +++ b/llvm/lib/IR/Intrinsics.cpp @@ -30,10 +30,55 @@ #include "llvm/IR/IntrinsicsXCore.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" +#include "llvm/Support/IntrinsicID.h" using namespace llvm; using namespace Intrinsic; +/// Table of per-target intrinsic name tables. +#define GET_INTRINSIC_TARGET_DATA +#include "llvm/IR/IntrinsicImpl.inc" +#undef GET_INTRINSIC_TARGET_DATA +size_t constexpr NumTargets = sizeof(TargetInfos) / sizeof(TargetInfos[0]); + +// Returns true if the given intrinsic ID is valid, that is, its value is one +// of the enum values defined for this intrinsic (including not_intrinsic). +bool Intrinsic::IsIntrinsicIDValid(ID ID) { + if (ID == not_intrinsic) + return true; + auto Decoded = decodeIntrinsicIDNoFail(ID); + if (!Decoded) + return false; + unsigned TargetIdx = Decoded->first; + unsigned IntrinsicIdx = Decoded->second; + return TargetIdx < NumTargets && IntrinsicIdx < TargetInfos[TargetIdx].Count; +} + +// Returns linear index of ID if its valid, else returns 0. +unsigned getLinearIndex(ID ID) { + auto Decoded = decodeIntrinsicIDNoFail(ID); + if (!Decoded) + return 0; + unsigned TargetIdx = Decoded->first; + unsigned IntrinsicIdx = Decoded->second; + return TargetInfos[TargetIdx].FirstLinearIndex + IntrinsicIdx; +} + +ID Intrinsic::GetNextValidIntrinsicID(ID ID) { + if (ID == not_intrinsic) + return encodeIntrinsicID(0, 0); + if (ID == last_valid_intrinsic_id) + return end_id; + if (ID == end_id) + llvm_unreachable("Cannot find the next valid intrisnic"); + auto [TargetIndex, IntrinsicIndex] = decodeIntrinsicID(ID); + if (IntrinsicIndex + 1 < TargetInfos[TargetIndex].Count) + return encodeIntrinsicID(TargetIndex, IntrinsicIndex + 1); + if (TargetIndex + 1 < NumTargets) + return encodeIntrinsicID(TargetIndex + 1, 0); + llvm_unreachable("Cannot find the next valid intrisnic"); +} + /// Table of string intrinsic names indexed by enum value. static constexpr const char *const IntrinsicNameTable[] = { "not_intrinsic", @@ -43,12 +88,12 @@ static constexpr const char *const IntrinsicNameTable[] = { }; StringRef Intrinsic::getBaseName(ID id) { - assert(id < num_intrinsics && "Invalid intrinsic ID!"); - return IntrinsicNameTable[id]; + assert(IsIntrinsicIDValid(id) && "Invalid intrinsic ID!"); + return ArrayRef(IntrinsicNameTable)[getLinearIndex(id)]; } StringRef Intrinsic::getName(ID id) { - assert(id < num_intrinsics && "Invalid intrinsic ID!"); + assert(IsIntrinsicIDValid(id) && "Invalid intrinsic ID!"); assert(!isOverloaded(id) && "This version of getName does not support overloading"); return getBaseName(id); @@ -155,8 +200,7 @@ static std::string getMangledTypeStr(Type *Ty, bool &HasUnnamedType) { static std::string getIntrinsicNameImpl(ID Id, ArrayRef Tys, Module *M, FunctionType *FT, bool EarlyModuleCheck) { - - assert(Id < num_intrinsics && "Invalid intrinsic ID!"); + assert(IsIntrinsicIDValid(Id) && "Invalid intrinsic ID!"); assert((Tys.empty() || isOverloaded(Id)) && "This version of getName is for overloaded intrinsics only"); (void)EarlyModuleCheck; @@ -445,11 +489,15 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef Infos, #undef GET_INTRINSIC_GENERATOR_GLOBAL void Intrinsic::getIntrinsicInfoTableEntries( - ID id, SmallVectorImpl &T) { + ID IntrinsicID, SmallVectorImpl &T) { static_assert(sizeof(IIT_Table[0]) == 2, "Expect 16-bit entries in IIT_Table"); + assert(IsIntrinsicIDValid(IntrinsicID)); + unsigned Idx = getLinearIndex(IntrinsicID); + if (Idx == 0) + return; // Check to see if the intrinsic's type was expressible by the table. - uint16_t TableVal = IIT_Table[id - 1]; + uint16_t TableVal = IIT_Table[Idx - 1]; // Decode the TableVal into an array of IITValues. SmallVector IITValues; @@ -602,18 +650,21 @@ FunctionType *Intrinsic::getType(LLVMContext &Context, ID id, return FunctionType::get(ResultTy, ArgTys, false); } -bool Intrinsic::isOverloaded(ID id) { +// Check if an intrinsic is overloaded or not using its linear index. +static bool isOverloadedUsingLinearIndex(unsigned Idx) { #define GET_INTRINSIC_OVERLOAD_TABLE #include "llvm/IR/IntrinsicImpl.inc" #undef GET_INTRINSIC_OVERLOAD_TABLE } -/// Table of per-target intrinsic name tables. -#define GET_INTRINSIC_TARGET_DATA -#include "llvm/IR/IntrinsicImpl.inc" -#undef GET_INTRINSIC_TARGET_DATA +bool Intrinsic::isOverloaded(ID id) { + assert(IsIntrinsicIDValid(id)); + return isOverloadedUsingLinearIndex(getLinearIndex(id)); +} -bool Intrinsic::isTargetIntrinsic(ID IID) { return IID > TargetInfos[0].Count; } +bool Intrinsic::isTargetIntrinsic(ID IID) { + return IID != not_intrinsic && decodeIntrinsicID(IID).first != 0; +} int Intrinsic::lookupLLVMIntrinsicByName(ArrayRef NameTable, StringRef Name, StringRef Target) { @@ -673,7 +724,29 @@ findTargetSubtable(StringRef Name) { // We've either found the target or just fall back to the generic set, which // is always first. const auto &TI = It != Targets.end() && It->Name == Target ? *It : Targets[0]; - return {ArrayRef(&IntrinsicNameTable[1] + TI.Offset, TI.Count), TI.Name}; + unsigned LinearIndex = TI.FirstLinearIndex; + return {ArrayRef(IntrinsicNameTable + LinearIndex, TI.Count), TI.Name}; +} + +static Intrinsic::ID getIntrinsicIDFromIndex(unsigned Idx) { + if (Idx == 0) + return Intrinsic::not_intrinsic; + + auto It = + partition_point(TargetInfos, [Idx](const IntrinsicTargetInfo &Info) { + return Info.FirstLinearIndex + Info.Count < Idx; + }); + // Idx, if present, will be in the entry at It or It + 1. + if (It == std::end(TargetInfos)) + return Intrinsic::not_intrinsic; + unsigned TargetIndex = std::distance(std::begin(TargetInfos), It); + if (It->FirstLinearIndex <= Idx && Idx < It->FirstLinearIndex + It->Count) + return encodeIntrinsicID(TargetIndex, Idx - It->FirstLinearIndex); + ++It; + ++TargetIndex; + if (It->FirstLinearIndex <= Idx && Idx < It->FirstLinearIndex + It->Count) + return encodeIntrinsicID(TargetIndex, Idx - It->FirstLinearIndex); + return Intrinsic::not_intrinsic; } /// This does the actual lookup of an intrinsic ID which matches the given @@ -683,18 +756,18 @@ ID Intrinsic::lookupIntrinsicID(StringRef Name) { int Idx = lookupLLVMIntrinsicByName(NameTable, Name, Target); if (Idx == -1) return not_intrinsic; - - // Intrinsic IDs correspond to the location in IntrinsicNameTable, but we have - // an index into a sub-table. + const auto MatchSize = strlen(NameTable[Idx]); + // Adjust the index from sub-table index to index into the global table. int Adjust = NameTable.data() - IntrinsicNameTable; - ID Id = static_cast(Idx + Adjust); + Idx += Adjust; // If the intrinsic is not overloaded, require an exact match. If it is // overloaded, require either exact or prefix match. - const auto MatchSize = strlen(NameTable[Idx]); assert(Name.size() >= MatchSize && "Expected either exact or prefix match"); bool IsExactMatch = Name.size() == MatchSize; - return IsExactMatch || isOverloaded(Id) ? Id : not_intrinsic; + return IsExactMatch || isOverloadedUsingLinearIndex(Idx) + ? getIntrinsicIDFromIndex(Idx) + : not_intrinsic; } /// This defines the "Intrinsic::getAttributes(ID id)" method. @@ -1028,8 +1101,9 @@ bool Intrinsic::matchIntrinsicVarArg(bool isVarArg, bool Intrinsic::getIntrinsicSignature(ID ID, FunctionType *FT, SmallVectorImpl &ArgTys) { - if (!ID) + if (ID == Intrinsic::not_intrinsic) return false; + assert(IsIntrinsicIDValid(ID)); SmallVector Table; getIntrinsicInfoTableEntries(ID, Table); diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 0814380b18848..ceec17980adbb 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -7551,7 +7551,7 @@ static unsigned getIntrinsicID(const SDNode *N) { return Intrinsic::not_intrinsic; case ISD::INTRINSIC_WO_CHAIN: { unsigned IID = N->getConstantOperandVal(0); - if (IID < Intrinsic::num_intrinsics) + if (Intrinsic::IsIntrinsicIDValid(IID)) return IID; return Intrinsic::not_intrinsic; } diff --git a/llvm/lib/Target/X86/X86IntrinsicsInfo.h b/llvm/lib/Target/X86/X86IntrinsicsInfo.h index 86fd04046d16a..f5477ec2f8131 100644 --- a/llvm/lib/Target/X86/X86IntrinsicsInfo.h +++ b/llvm/lib/Target/X86/X86IntrinsicsInfo.h @@ -79,8 +79,7 @@ enum IntrinsicType : uint16_t { }; struct IntrinsicData { - - uint16_t Id; + unsigned Id; IntrinsicType Type; uint16_t Opc0; uint16_t Opc1; diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td index 365d0c9fbff49..8d96f703f9789 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td @@ -36,7 +36,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4([[L72:[0-9]+]]), // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled), // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, -// CHECK-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::1in_1out), +// CHECK-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode4(Intrinsic::1in_1out), // CHECK-NEXT: // MIs[0] a // CHECK-NEXT: // No operand predicates // CHECK-NEXT: // MIs[0] Operand 2 @@ -45,10 +45,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: // Combiner Rule #0: IntrinTest0 // CHECK-NEXT: GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC), // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define), -// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/0, GIMT_Encode2(Intrinsic::0in_1out), +// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/0, GIMT_Encode4(Intrinsic::0in_1out), // CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC), // CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/0, // a -// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/1, GIMT_Encode2(Intrinsic::1in_1out), +// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/1, GIMT_Encode4(Intrinsic::1in_1out), // CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/0, // CHECK-NEXT: GIR_EraseRootFromParent_Done, // CHECK-NEXT: // Label 3: @[[L72]] @@ -57,7 +57,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4([[L131:[0-9]+]]), // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled), // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, -// CHECK-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::sideeffects_1in_1out), +// CHECK-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode4(Intrinsic::sideeffects_1in_1out), // CHECK-NEXT: // MIs[0] a // CHECK-NEXT: // No operand predicates // CHECK-NEXT: // MIs[0] b @@ -66,11 +66,11 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: // Combiner Rule #1: SpecialIntrins // CHECK-NEXT: GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC_CONVERGENT), // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define), -// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/0, GIMT_Encode2(Intrinsic::convergent_1in_1out), +// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/0, GIMT_Encode4(Intrinsic::convergent_1in_1out), // CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/2, // b // CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS), // CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/0, // a -// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/1, GIMT_Encode2(Intrinsic::convergent_sideeffects_1in_1out), +// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/1, GIMT_Encode4(Intrinsic::convergent_sideeffects_1in_1out), // CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/0, // CHECK-NEXT: GIR_MergeMemOperands, /*InsnID*/1, /*NumInsns*/1, /*MergeInsnID's*/0, // CHECK-NEXT: GIR_EraseRootFromParent_Done, @@ -78,6 +78,6 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIM_Reject, // CHECK-NEXT: // Label 2: @[[L132]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: }; // Size: 125 bytes +// CHECK-NEXT: }; // Size: 137 bytes // CHECK-NEXT: return MatchTable0; // CHECK-NEXT: } diff --git a/llvm/test/TableGen/GlobalISelEmitter-SDNodeXForm-timm.td b/llvm/test/TableGen/GlobalISelEmitter-SDNodeXForm-timm.td index 8d6dedf2f920c..e4f0a3bc5f6cd 100644 --- a/llvm/test/TableGen/GlobalISelEmitter-SDNodeXForm-timm.td +++ b/llvm/test/TableGen/GlobalISelEmitter-SDNodeXForm-timm.td @@ -19,7 +19,7 @@ def SLEEP : I<(outs), (ins i32imm:$src0), []>; def FOO : I<(outs GPR32:$dst), (ins GPR32:$src0, i32imm:$src1), []>; // GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC), -// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::mytarget_foo), +// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode4(Intrinsic::mytarget_foo), // GISEL: GIM_CheckIsImm, /*MI*/0, /*Op*/3, // GISEL: GIR_CustomOperandRenderer, /*InsnID*/0, /*OldInsnID*/0, /*OpIdx*/3, /*OperandRenderer*/GIMT_Encode2(GICR_renderShiftImml1), // src1 def : Pat< @@ -28,7 +28,7 @@ def : Pat< >; // GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS), -// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode2(Intrinsic::mytarget_sleep), +// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode4(Intrinsic::mytarget_sleep), // GISEL: GIM_CheckIsImm, /*MI*/0, /*Op*/1, // GISEL: GIR_CustomOperandRenderer, /*InsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, /*OperandRenderer*/GIMT_Encode2(GICR_renderShiftImml1), // src0 def : Pat< diff --git a/llvm/test/TableGen/GlobalISelEmitter-immarg-literal-pattern.td b/llvm/test/TableGen/GlobalISelEmitter-immarg-literal-pattern.td index 6b4012eb736cb..2c9bd7ae3a049 100644 --- a/llvm/test/TableGen/GlobalISelEmitter-immarg-literal-pattern.td +++ b/llvm/test/TableGen/GlobalISelEmitter-immarg-literal-pattern.td @@ -23,7 +23,7 @@ def CAT1 : I<(outs GPR32:$dst), (ins GPR32:$src0), []>; // Make sure there is no type check. // GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS), -// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode2(Intrinsic::mytarget_sleep), +// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode4(Intrinsic::mytarget_sleep), // GISEL-NEXT: // MIs[0] Operand 1 // GISEL-NEXT: GIM_CheckLiteralInt, /*MI*/0, /*Op*/1, GIMT_Encode8(0), def : Pat< @@ -32,7 +32,7 @@ def : Pat< >; // GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS), -// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode2(Intrinsic::mytarget_sleep), +// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode4(Intrinsic::mytarget_sleep), // GISEL-NEXT: // MIs[0] Operand 1 // GISEL-NEXT: GIM_CheckLiteralInt, /*MI*/0, /*Op*/1, GIMT_Encode8(1), def : Pat< diff --git a/llvm/test/TableGen/GlobalISelEmitter-input-discard.td b/llvm/test/TableGen/GlobalISelEmitter-input-discard.td index 202ff4a5758d7..fbfa6707d1e7b 100644 --- a/llvm/test/TableGen/GlobalISelEmitter-input-discard.td +++ b/llvm/test/TableGen/GlobalISelEmitter-input-discard.td @@ -10,7 +10,7 @@ def FOO : I<(outs GPR32:$dst), (ins GPR32Op:$src0, GPR32Op:$src1), []>; // GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS), // GISEL-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/4, -// GISEL-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::tgt_foo), +// GISEL-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode4(Intrinsic::tgt_foo), // GISEL-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32, // GISEL-NEXT: GIM_RootCheckType, /*Op*/2, /*Type*/GILLT_s32, // GISEL-NEXT: GIM_RootCheckType, /*Op*/3, /*Type*/GILLT_s32, diff --git a/llvm/test/TableGen/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter.td index 7dbaf4390c0f7..fb6c0a3f5006e 100644 --- a/llvm/test/TableGen/GlobalISelEmitter.td +++ b/llvm/test/TableGen/GlobalISelEmitter.td @@ -513,7 +513,7 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3), // R00O-NEXT: GIM_Reject, // R00O: // Label [[DEFAULT_NUM]]: @[[DEFAULT]] // R00O-NEXT: GIM_Reject, -// R00O-NEXT: }; // Size: 1832 bytes +// R00O-NEXT: }; // Size: 1834 bytes def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, GPR32:$src4), [(set GPR32:$dst, @@ -534,7 +534,7 @@ def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, G // R01C-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]), // Rule ID 1 // // R01C-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // -// R01O-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::mytarget_nop), +// R01O-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode4(Intrinsic::mytarget_nop), // R01O-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32, // R01O-NEXT: GIM_RootCheckType, /*Op*/2, /*Type*/GILLT_s32, // R01O-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), @@ -544,7 +544,7 @@ def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, G // R01N-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32, // R01N-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), // R01N-NEXT: // MIs[0] Operand 1 -// R01N-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::mytarget_nop), +// R01N-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode4(Intrinsic::mytarget_nop), // R01N-NEXT: // MIs[0] src1 // R01N-NEXT: GIM_RootCheckType, /*Op*/2, /*Type*/GILLT_s32, // @@ -1206,5 +1206,5 @@ def BR : I<(outs), (ins unknown:$target), [(br bb:$target)]>; // NOOPT-NEXT: GIM_Reject, -// NOOPT-NEXT: }; // Size: 1459 bytes +// NOOPT-NEXT: }; // Size: 1461 bytes // NOOPT-NEXT: return MatchTable0; diff --git a/llvm/test/TableGen/GlobalISelEmitterOverloadedPtr.td b/llvm/test/TableGen/GlobalISelEmitterOverloadedPtr.td index 422edbba0e7a0..c57da4ebe0dbd 100644 --- a/llvm/test/TableGen/GlobalISelEmitterOverloadedPtr.td +++ b/llvm/test/TableGen/GlobalISelEmitterOverloadedPtr.td @@ -11,7 +11,7 @@ let TargetPrefix = "mytarget" in { // Ensure that llvm_anyptr_ty on an intrinsic results in a // GIM_CheckPointerToAny rather than a GIM_CheckType. // -// CHECK: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::mytarget_anyptr), +// CHECK: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode4(Intrinsic::mytarget_anyptr), // CHECK-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), // CHECK-NEXT: // MIs[0] src diff --git a/llvm/test/TableGen/immarg-predicated.td b/llvm/test/TableGen/immarg-predicated.td index dcacb2f8f1de3..31f5bf49c7d3d 100644 --- a/llvm/test/TableGen/immarg-predicated.td +++ b/llvm/test/TableGen/immarg-predicated.td @@ -9,7 +9,7 @@ def int_mytarget_sleep0 : Intrinsic<[], [llvm_i32_ty], [ImmArg>]>; // GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS), // GISEL-NEXT: // MIs[0] Operand 0 -// GISEL-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode2(Intrinsic::mytarget_sleep0), +// GISEL-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode4(Intrinsic::mytarget_sleep0), // GISEL-NEXT: // MIs[0] src // GISEL-NEXT: GIM_CheckIsImm, /*MI*/0, /*Op*/1, // GISEL-NEXT: GIM_CheckImmOperandPredicate, /*MI*/0, /*MO*/1, /*Predicate*/GIMT_Encode2(GICXXPred_I64_Predicate_tuimm9), diff --git a/llvm/test/TableGen/immarg.td b/llvm/test/TableGen/immarg.td index e5fd06ce6c083..625b5d46a4ef6 100644 --- a/llvm/test/TableGen/immarg.td +++ b/llvm/test/TableGen/immarg.td @@ -10,7 +10,7 @@ def int_mytarget_sleep1 : Intrinsic<[], [llvm_i32_ty], [ImmArg>]>; // GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS), // GISEL-NEXT: // MIs[0] Operand 0 -// GISEL-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode2(Intrinsic::mytarget_sleep0), +// GISEL-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode4(Intrinsic::mytarget_sleep0), // GISEL-NEXT: // MIs[0] src // GISEL-NEXT: GIM_CheckIsImm, /*MI*/0, /*Op*/1, // GISEL-NEXT: // (intrinsic_void {{[0-9]+}}:{ *:[iPTR] }, (timm:{ *:[i32] }):$src) => (SLEEP0 (timm:{ *:[i32] }):$src) diff --git a/llvm/test/TableGen/intrinsic-overload-conflict.td b/llvm/test/TableGen/intrinsic-overload-conflict.td index 84333119d41f5..788c49acde8ab 100644 --- a/llvm/test/TableGen/intrinsic-overload-conflict.td +++ b/llvm/test/TableGen/intrinsic-overload-conflict.td @@ -3,7 +3,7 @@ include "llvm/IR/Intrinsics.td" -// CHECK: foo = 1, +// CHECK: foo = {{.+}}, def int_foo : Intrinsic<[llvm_any_ty]>; // No conflicts, since .bar is not a vaid mangled type. diff --git a/llvm/test/TableGen/intrinsic-struct.td b/llvm/test/TableGen/intrinsic-struct.td index 467fd9057c183..10e98e589b8e4 100644 --- a/llvm/test/TableGen/intrinsic-struct.td +++ b/llvm/test/TableGen/intrinsic-struct.td @@ -7,7 +7,7 @@ include "llvm/IR/Intrinsics.td" // Make sure we can return up to 9 values. -// CHECK-ENUM: returns_9_results = {{[0-9]+}}, // llvm.returns.9.results +// CHECK-ENUM: returns_9_results = {{.+}}, // llvm.returns.9.results def int_returns_9_results : Intrinsic< !listsplat(llvm_anyint_ty, 9), [], [], "llvm.returns.9.results">; diff --git a/llvm/test/TableGen/predicate-patfags.td b/llvm/test/TableGen/predicate-patfags.td index 39133f324f305..42dfd700f18a3 100644 --- a/llvm/test/TableGen/predicate-patfags.td +++ b/llvm/test/TableGen/predicate-patfags.td @@ -55,12 +55,12 @@ def TGTmul24_oneuse : PatFrag< // SBUILTIN: if (!SDValue(N, 0).hasOneUse()) return false; // GISEL: GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS), -// GISEL: GIM_CheckIntrinsicID, /*MI*/1, /*Op*/1, GIMT_Encode2(Intrinsic::tgt_mul24), +// GISEL: GIM_CheckIntrinsicID, /*MI*/1, /*Op*/1, GIMT_Encode4(Intrinsic::tgt_mul24), // GBUILTIN: GIM_CheckHasOneUse, /*MI*/1, // GCUSTOM: GIM_CheckCxxInsnPredicate, /*MI*/1, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_TGTmul24_oneuse), // GISEL: GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS), -// GISEL: GIM_CheckIntrinsicID, /*MI*/1, /*Op*/1, GIMT_Encode2(Intrinsic::tgt_mul24), +// GISEL: GIM_CheckIntrinsicID, /*MI*/1, /*Op*/1, GIMT_Encode4(Intrinsic::tgt_mul24), // GBUILTIN: GIM_CheckHasOneUse, /*MI*/1, // GCUSTOM: GIM_CheckCxxInsnPredicate, /*MI*/1, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_TGTmul24_oneuse), diff --git a/llvm/unittests/IR/VPIntrinsicTest.cpp b/llvm/unittests/IR/VPIntrinsicTest.cpp index d6ad7599ce461..48a0dc86724d4 100644 --- a/llvm/unittests/IR/VPIntrinsicTest.cpp +++ b/llvm/unittests/IR/VPIntrinsicTest.cpp @@ -370,27 +370,28 @@ TEST_F(VPIntrinsicTest, IntrinsicIDRoundTrip) { /// Check that going from intrinsic to VP intrinsic and back results in the same /// intrinsic. TEST_F(VPIntrinsicTest, IntrinsicToVPRoundTrip) { + using namespace Intrinsic; bool IsFullTrip = false; - Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic + 1; - for (; IntrinsicID < Intrinsic::num_intrinsics; IntrinsicID++) { - Intrinsic::ID VPID = VPIntrinsic::getForIntrinsic(IntrinsicID); + for (ID IID = GetNextValidIntrinsicID(not_intrinsic); IID != end_id; + IID = GetNextValidIntrinsicID(IID)) { + ID VPID = VPIntrinsic::getForIntrinsic(IID); // No equivalent VP intrinsic available. if (VPID == Intrinsic::not_intrinsic) continue; // Return itself if passed intrinsic ID is VP intrinsic. - if (VPIntrinsic::isVPIntrinsic(IntrinsicID)) { - ASSERT_EQ(IntrinsicID, VPID); + if (VPIntrinsic::isVPIntrinsic(IID)) { + ASSERT_EQ(IID, VPID); continue; } - std::optional RoundTripIntrinsicID = + std::optional RoundTripIntrinsicID = VPIntrinsic::getFunctionalIntrinsicIDForVP(VPID); // No equivalent non-predicated intrinsic available. if (!RoundTripIntrinsicID) continue; - ASSERT_EQ(*RoundTripIntrinsicID, IntrinsicID); + ASSERT_EQ(*RoundTripIntrinsicID, IID); IsFullTrip = true; } ASSERT_TRUE(IsFullTrip); diff --git a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp index 18e0b8fd135bb..1ecf46edf7909 100644 --- a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp +++ b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp @@ -13,8 +13,12 @@ #include "CodeGenIntrinsics.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringSet.h" #include "llvm/ADT/Twine.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/IntrinsicID.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" #include @@ -44,35 +48,45 @@ CodeGenIntrinsicTable::CodeGenIntrinsicTable(const RecordKeeper &RC) { CodeGenIntrinsicContext Ctx(RC); ArrayRef Defs = RC.getAllDerivedDefinitions("Intrinsic"); - Intrinsics.reserve(Defs.size()); - - for (const Record *Def : Defs) - Intrinsics.emplace_back(CodeGenIntrinsic(Def, Ctx)); - - llvm::sort(Intrinsics, - [](const CodeGenIntrinsic &LHS, const CodeGenIntrinsic &RHS) { - // Order target independent intrinsics before target dependent - // ones. - bool LHSHasTarget = !LHS.TargetPrefix.empty(); - bool RHSHasTarget = !RHS.TargetPrefix.empty(); - - // To ensure deterministic sorted order when duplicates are - // present, use record ID as a tie-breaker similar to - // sortAndReportDuplicates in Utils.cpp. - unsigned LhsID = LHS.TheDef->getID(); - unsigned RhsID = RHS.TheDef->getID(); - - return std::tie(LHSHasTarget, LHS.Name, LhsID) < - std::tie(RHSHasTarget, RHS.Name, RhsID); - }); - - Targets.push_back({"", 0, 0}); - for (size_t I = 0, E = Intrinsics.size(); I < E; ++I) - if (Intrinsics[I].TargetPrefix != Targets.back().Name) { - Targets.back().Count = I - Targets.back().Offset; - Targets.push_back({Intrinsics[I].TargetPrefix, I, 0}); - } - Targets.back().Count = Intrinsics.size() - Targets.back().Offset; + + // Bucket each intrinsic into a per-target list of intrinsics. Use std::map + // so that the targets sorted by name when we iterate over the map. + std::map TargetMap; + + // Always create entry for target independent intrinsics. + TargetMap[""].Name = ""; + + for (const Record *Def : Defs) { + CodeGenIntrinsic Int(Def, Ctx); + TargetMap[Int.TargetPrefix].Intrinsics.push_back(Int); + } + + auto IntrinsicCmp = [](const CodeGenIntrinsic &LHS, + const CodeGenIntrinsic &RHS) { + // To ensure deterministic sorted order when duplicates are present, use + // record ID as a tie-breaker similar to sortAndReportDuplicates in + // Utils.cpp. + unsigned LhsID = LHS.TheDef->getID(); + unsigned RhsID = RHS.TheDef->getID(); + + return std::tie(LHS.Name, LhsID) < std::tie(RHS.Name, RhsID); + }; + + // Sort intrinsics by name within each target, and collect all targets + // (alreay sorted by target name). Also assign linear index to all of them. + // Linear index 0 is reserved for not_intrinsic. + unsigned LinearIndex = 1; + unsigned TargetIndex = 0; + for (auto &[TargetName, TSet] : TargetMap) { + Targets.push_back(std::move(TSet)); + TargetSet &T = Targets.back(); + T.Name = TargetName; + T.TargetIndex = TargetIndex++; + llvm::sort(T.Intrinsics, IntrinsicCmp); + T.FirstLinearIndex = LinearIndex; + LinearIndex += T.Intrinsics.size(); + } + NumIntrinsics = LinearIndex - 1; CheckDuplicateIntrinsics(); CheckTargetIndependentIntrinsics(); @@ -87,20 +101,23 @@ void CodeGenIntrinsicTable::CheckDuplicateIntrinsics() const { // there cannot be be duplicate as TableGen parser would have flagged that. // However, if the name was specified in the intrinsic definition, then its // possible to have duplicate names. - auto I = std::adjacent_find( - Intrinsics.begin(), Intrinsics.end(), - [](const CodeGenIntrinsic &Int1, const CodeGenIntrinsic &Int2) { - return Int1.Name == Int2.Name; - }); - if (I == Intrinsics.end()) - return; - - // Found a duplicate intrinsics. - const CodeGenIntrinsic &First = *I; - const CodeGenIntrinsic &Second = *(I + 1); - PrintError(Second.TheDef, - Twine("Intrinsic `") + First.Name + "` is already defined"); - PrintFatalNote(First.TheDef, "Previous definition here"); + for (const TargetSet &T : getAllTargets()) { + ArrayRef Intrinsics = T.getIntrinsics(); + auto I = std::adjacent_find( + Intrinsics.begin(), Intrinsics.end(), + [](const CodeGenIntrinsic &Int1, const CodeGenIntrinsic &Int2) { + return Int1.Name == Int2.Name; + }); + if (I == Intrinsics.end()) + return; + + // Found a duplicate intrinsics. + const CodeGenIntrinsic &First = *I; + const CodeGenIntrinsic &Second = *(I + 1); + PrintError(Second.TheDef, + Twine("Intrinsic `") + First.Name + "` is already defined"); + PrintFatalNote(First.TheDef, "Previous definition here"); + } } // For target independent intrinsics, check that their second dotted component @@ -111,8 +128,7 @@ void CodeGenIntrinsicTable::CheckTargetIndependentIntrinsics() const { TargetNames.insert(Target.Name); // Set of target independent intrinsics. - const auto &Set = Targets[0]; - for (const auto &Int : ArrayRef(&Intrinsics[Set.Offset], Set.Count)) { + for (const auto &Int : Targets[0].getIntrinsics()) { StringRef Name = Int.Name; StringRef Prefix = Name.drop_front(5).split('.').first; if (!TargetNames.contains(Prefix)) @@ -204,7 +220,7 @@ static bool doesSuffixLookLikeMangledType(StringRef Suffix) { void CodeGenIntrinsicTable::CheckOverloadSuffixConflicts() const { for (const TargetSet &Set : Targets) { const CodeGenIntrinsic *Overloaded = nullptr; - for (const CodeGenIntrinsic &Int : (*this)[Set]) { + for (const CodeGenIntrinsic &Int : Set.getIntrinsics()) { // If we do not have an overloaded intrinsic to check against, nothing // to do except potentially identifying this as a candidate for checking // against in future iteration. @@ -248,6 +264,57 @@ void CodeGenIntrinsicTable::CheckOverloadSuffixConflicts() const { } } +// Enumerate all intrinsics and call back the visitor function with the +// intrinsic and its index. +void CodeGenIntrinsicTable::enumerateIntrinsics( + CodeGenIntrinsicTable::IntrinsicVisitor Visitor) const { + for (const TargetSet &T : getAllTargets()) { + unsigned Idx = T.FirstLinearIndex - 1; + for (const CodeGenIntrinsic &Int : T.getIntrinsics()) + Visitor(Idx++, Int); + } +} + +unsigned CodeGenIntrinsicTable::getIntrinsicID(const Record *Def) const { + for (const auto &T : getAllTargets()) + for (const auto &[IntrinsicIndex, Int] : enumerate(T.getIntrinsics())) + if (Int.TheDef == Def) + return Intrinsic::encodeIntrinsicID(T.TargetIndex, IntrinsicIndex); + errs() << "Cannot find intrinsic for record: " << Def->getName() << '\n'; + llvm_unreachable("Unknown intrinsic!"); +} + +const CodeGenIntrinsic & +CodeGenIntrinsicTable::getIntrinsic(const Record *Def) const { + for (const TargetSet &T : getAllTargets()) + for (const CodeGenIntrinsic &Int : T.getIntrinsics()) + if (Int.TheDef == Def) + return Int; + errs() << "Cannot find intrinsic for record: " << Def->getName() << '\n'; + llvm_unreachable("Unknown intrinsic!"); +} + +const CodeGenIntrinsic &CodeGenIntrinsicTable::getIntrinsic(unsigned ID) const { + auto getIntrinsicImpl = [&](unsigned ID) -> const CodeGenIntrinsic * { + auto Decoded = Intrinsic::decodeIntrinsicIDNoFail(ID); + if (!Decoded) + return nullptr; + unsigned TargetIndex = Decoded->first; + unsigned IntrinsicIndex = Decoded->second; + if (TargetIndex >= getAllTargets().size()) + return nullptr; + const TargetSet &T = getAllTargets()[TargetIndex]; + ArrayRef Ints = T.getIntrinsics(); + if (IntrinsicIndex >= Ints.size()) + return nullptr; + return &Ints[IntrinsicIndex]; + }; + if (const CodeGenIntrinsic *Result = getIntrinsicImpl(ID)) + return *Result; + errs() << "Cannot find intrinsic for ID: 0x" << Twine::utohexstr(ID) << '\n'; + llvm_unreachable("Unknown intrinsic!"); +} + const CodeGenIntrinsic &CodeGenIntrinsicMap::operator[](const Record *Record) { if (!Record->isSubClassOf("Intrinsic")) PrintFatalError("Intrinsic defs should be subclass of 'Intrinsic' class"); diff --git a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h index 8428d09a94009..bd77d666ab45a 100644 --- a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h +++ b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h @@ -16,6 +16,7 @@ #include "SDNodeProperties.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Support/ModRef.h" #include @@ -172,31 +173,35 @@ class CodeGenIntrinsicTable { public: struct TargetSet { StringRef Name; - size_t Offset; - size_t Count; + unsigned TargetIndex; + std::vector Intrinsics; + size_t FirstLinearIndex; + + ArrayRef getIntrinsics() const { return Intrinsics; } }; explicit CodeGenIntrinsicTable(const RecordKeeper &RC); - bool empty() const { return Intrinsics.empty(); } - size_t size() const { return Intrinsics.size(); } - auto begin() const { return Intrinsics.begin(); } - auto end() const { return Intrinsics.end(); } - const CodeGenIntrinsic &operator[](size_t Pos) const { - return Intrinsics[Pos]; - } - ArrayRef operator[](const TargetSet &Set) const { - return ArrayRef(&Intrinsics[Set.Offset], Set.Count); - } - ArrayRef getTargets() const { return Targets; } + ArrayRef getAllTargets() const { return Targets; } + + using IntrinsicVisitor = + function_ref; + void enumerateIntrinsics(IntrinsicVisitor Visitor) const; + + unsigned getNumIntrinsics() const { return NumIntrinsics; } + + // Find the intrinsic corresponding to the given record. + unsigned getIntrinsicID(const Record *Def) const; + const CodeGenIntrinsic &getIntrinsic(const Record *Def) const; + const CodeGenIntrinsic &getIntrinsic(unsigned ID) const; private: void CheckDuplicateIntrinsics() const; void CheckTargetIndependentIntrinsics() const; void CheckOverloadSuffixConflicts() const; - std::vector Intrinsics; std::vector Targets; + unsigned NumIntrinsics; }; // This class builds `CodeGenIntrinsic` on demand for a given Def. diff --git a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp index f17c62dd1fd9d..95647ebd76c76 100644 --- a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp +++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp @@ -2986,7 +2986,7 @@ TreePatternNodePtr TreePattern::ParseTreePattern(const Init *TheInit, // convert the intrinsic name to a number. if (Operator->isSubClassOf("Intrinsic")) { const CodeGenIntrinsic &Int = getDAGPatterns().getIntrinsic(Operator); - unsigned IID = getDAGPatterns().getIntrinsicID(Operator) + 1; + unsigned IID = getDAGPatterns().getIntrinsicID(Operator); // If this intrinsic returns void, it must have side-effects and thus a // chain. diff --git a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h index f85753ff5ac80..0cfb07a07f54c 100644 --- a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h +++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h @@ -1158,23 +1158,15 @@ class CodeGenDAGPatterns { } const CodeGenIntrinsic &getIntrinsic(const Record *R) const { - for (unsigned i = 0, e = Intrinsics.size(); i != e; ++i) - if (Intrinsics[i].TheDef == R) - return Intrinsics[i]; - llvm_unreachable("Unknown intrinsic!"); + return Intrinsics.getIntrinsic(R); } const CodeGenIntrinsic &getIntrinsicInfo(unsigned IID) const { - if (IID - 1 < Intrinsics.size()) - return Intrinsics[IID - 1]; - llvm_unreachable("Bad intrinsic ID!"); + return Intrinsics.getIntrinsic(IID); } unsigned getIntrinsicID(const Record *R) const { - for (unsigned i = 0, e = Intrinsics.size(); i != e; ++i) - if (Intrinsics[i].TheDef == R) - return i; - llvm_unreachable("Unknown intrinsic!"); + return Intrinsics.getIntrinsicID(R); } const DAGDefaultOperand &getDefaultOperand(const Record *R) const { diff --git a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp index 15ec7e17130de..c386af133ea8d 100644 --- a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp +++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp @@ -1305,7 +1305,7 @@ void IntrinsicIDOperandMatcher::emitPredicateOpcodes(MatchTable &Table, Table << MatchTable::Opcode("GIM_CheckIntrinsicID") << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID) << MatchTable::Comment("Op") << MatchTable::ULEB128Value(OpIdx) - << MatchTable::NamedValue(2, "Intrinsic::" + II->EnumName.str()) + << MatchTable::NamedValue(4, "Intrinsic::" + II->EnumName.str()) << MatchTable::LineBreak; } @@ -2095,7 +2095,7 @@ void IntrinsicIDRenderer::emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const { Table << MatchTable::Opcode("GIR_AddIntrinsicID") << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnID) - << MatchTable::NamedValue(2, "Intrinsic::" + II->EnumName.str()) + << MatchTable::NamedValue(4, "Intrinsic::" + II->EnumName.str()) << MatchTable::LineBreak; } diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp index 070d7522a97be..612876e72b1fe 100644 --- a/llvm/utils/TableGen/IntrinsicEmitter.cpp +++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp @@ -19,6 +19,7 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/IntrinsicID.h" #include "llvm/Support/ModRef.h" #include "llvm/Support/raw_ostream.h" #include "llvm/TableGen/Error.h" @@ -35,6 +36,7 @@ #include #include using namespace llvm; +using TargetSet = CodeGenIntrinsicTable::TargetSet; static cl::OptionCategory GenIntrinsicCat("Options for -gen-intrinsic-enums"); static cl::opt @@ -87,8 +89,10 @@ void IntrinsicEmitter::run(raw_ostream &OS, bool Enums) { // Emit the enum information. EmitEnumInfo(Ints, OS); - // Emit ArgKind for Intrinsics.h. - EmitArgKind(OS); + if (IntrinsicPrefix.empty()) { + // Emit ArgKind for Intrinsics.h. + EmitArgKind(OS); + } } else { // Emit IIT_Info constants. EmitIITInfo(OS); @@ -121,9 +125,8 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints, // Find the TargetSet for which to generate enums. There will be an initial // set with an empty target prefix which will include target independent // intrinsics like dbg.value. - using TargetSet = CodeGenIntrinsicTable::TargetSet; const TargetSet *Set = nullptr; - for (const auto &Target : Ints.getTargets()) { + for (const auto &[Idx, Target] : enumerate(Ints.getAllTargets())) { if (Target.Name == IntrinsicPrefix) { Set = &Target; break; @@ -131,7 +134,7 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints, } if (!Set) { // The first entry is for target independent intrinsics, so drop it. - auto KnowTargets = Ints.getTargets().drop_front(); + auto KnowTargets = Ints.getAllTargets().drop_front(); PrintFatalError([KnowTargets](raw_ostream &OS) { OS << "tried to generate intrinsics for unknown target " << IntrinsicPrefix << "\nKnown targets are: "; @@ -142,10 +145,10 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints, } // Generate a complete header for target specific intrinsics. + std::string UpperPrefix = StringRef(IntrinsicPrefix).upper(); if (IntrinsicPrefix.empty()) { OS << "#ifdef GET_INTRINSIC_ENUM_VALUES\n"; } else { - std::string UpperPrefix = StringRef(IntrinsicPrefix).upper(); OS << formatv("#ifndef LLVM_IR_INTRINSIC_{}_ENUMS_H\n", UpperPrefix); OS << formatv("#define LLVM_IR_INTRINSIC_{}_ENUMS_H\n", UpperPrefix); OS << "namespace llvm::Intrinsic {\n"; @@ -154,13 +157,13 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints, OS << "// Enum values for intrinsics.\n"; bool First = true; - for (const auto &Int : Ints[*Set]) { + for (const auto &Int : Set->getIntrinsics()) { OS << " " << Int.EnumName; // Assign a value to the first intrinsic in this target set so that all // intrinsic ids are distinct. if (First) { - OS << " = " << Set->Offset + 1; + OS << formatv(" = ({} << 16) + 1", Set->TargetIndex); First = false; } @@ -170,22 +173,23 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints, OS << formatv(" // {}\n", Int.Name); } - // Emit num_intrinsics into the target neutral enum. if (IntrinsicPrefix.empty()) { - OS << formatv(" num_intrinsics = {}\n", Ints.size() + 1); + const TargetSet &LastTarget = Ints.getAllTargets().back(); + const size_t LastCount = LastTarget.getIntrinsics().size(); + OS << formatv(" last_valid_intrinsic_id = ({} << 16) + {} + 1,\n", + LastTarget.TargetIndex, LastCount - 1); OS << "#endif\n\n"; } else { - OS << R"(}; // enum + OS << formatv(R"(}; // enum {}Intrinsics } // namespace llvm::Intrinsic #endif -)"; +)", + UpperPrefix); } } void IntrinsicEmitter::EmitArgKind(raw_ostream &OS) { - if (!IntrinsicPrefix.empty()) - return; OS << "// llvm::Intrinsic::IITDescriptor::ArgKind.\n"; OS << "#ifdef GET_INTRINSIC_ARGKIND\n"; if (const auto RecArgKind = Records.getDef("ArgKind")) { @@ -224,13 +228,14 @@ void IntrinsicEmitter::EmitTargetInfo(const CodeGenIntrinsicTable &Ints, #ifdef GET_INTRINSIC_TARGET_DATA struct IntrinsicTargetInfo { StringLiteral Name; - size_t Offset; - size_t Count; + unsigned Count; + unsigned FirstLinearIndex; }; static constexpr IntrinsicTargetInfo TargetInfos[] = { )"; - for (const auto [Name, Offset, Count] : Ints.getTargets()) - OS << formatv(" {{\"{}\", {}, {}},\n", Name, Offset, Count); + for (const TargetSet &T : Ints.getAllTargets()) + OS << formatv(" {{\"{}\", {}, {}},\n", T.Name, T.Intrinsics.size(), + T.FirstLinearIndex); OS << R"(}; #endif @@ -243,30 +248,30 @@ void IntrinsicEmitter::EmitIntrinsicToNameTable( #ifdef GET_INTRINSIC_NAME_TABLE // Note that entry #0 is the invalid intrinsic! )"; - for (const auto &Int : Ints) + Ints.enumerateIntrinsics([&OS](unsigned, const CodeGenIntrinsic &Int) { OS << " \"" << Int.Name << "\",\n"; + }); OS << "#endif\n\n"; } void IntrinsicEmitter::EmitIntrinsicToOverloadTable( const CodeGenIntrinsicTable &Ints, raw_ostream &OS) { - OS << R"(// Intrinsic ID to overload bitset. + OS << formatv(R"(// Intrinsic ID to overload bitset. #ifdef GET_INTRINSIC_OVERLOAD_TABLE -static constexpr uint8_t OTable[] = { - 0 - )"; - for (auto [I, Int] : enumerate(Ints)) { - // Add one to the index so we emit a null bit for the invalid #0 intrinsic. - size_t Idx = I + 1; - - if (Idx % 8 == 0) +static constexpr uint8_t OTable[] = {{ + 0)"); + Ints.enumerateIntrinsics([&OS](unsigned Idx, const CodeGenIntrinsic &Int) { + // Add one to the index so we emit a null bit for the invalid #0 + // intrinsic. + unsigned LinearIndex = Idx + 1; + if (LinearIndex % 8 == 0) OS << ",\n 0"; if (Int.isOverloaded) - OS << " | (1<<" << Idx % 8 << ')'; - } + OS << " | (1<<" << LinearIndex % 8 << ')'; + }); OS << "\n};\n\n"; // OTable contains a true bit at the position if the intrinsic is overloaded. - OS << "return (OTable[id/8] & (1 << (id%8))) != 0;\n"; + OS << "return (ArrayRef(OTable)[Idx/8] & (1 << (Idx%8))) != 0;\n"; OS << "#endif\n\n"; } @@ -316,10 +321,10 @@ void IntrinsicEmitter::EmitGenerator(const CodeGenIntrinsicTable &Ints, std::vector FixedEncodings; SequenceToOffsetTable LongEncodingTable; - FixedEncodings.reserve(Ints.size()); + FixedEncodings.reserve(Ints.getNumIntrinsics()); // Compute the unique argument type info. - for (const CodeGenIntrinsic &Int : Ints) { + Ints.enumerateIntrinsics([&](unsigned, const CodeGenIntrinsic &Int) { // Get the signature for the intrinsic. TypeSigTy TypeSig = ComputeTypeSignature(Int); @@ -327,14 +332,14 @@ void IntrinsicEmitter::EmitGenerator(const CodeGenIntrinsicTable &Ints, std::optional Result = encodePacked(TypeSig); if (Result && (*Result & Mask) == Result) { FixedEncodings.push_back(static_cast(*Result)); - continue; + return; } LongEncodingTable.add(TypeSig); // This is a placehold that we'll replace after the table is laid out. FixedEncodings.push_back(static_cast(~0U)); - } + }); LongEncodingTable.layout(); @@ -345,24 +350,25 @@ static constexpr {} IIT_Table[] = {{ FixedEncodingTypeName); unsigned MaxOffset = 0; - for (auto [Idx, FixedEncoding, Int] : enumerate(FixedEncodings, Ints)) { + Ints.enumerateIntrinsics([&](unsigned Idx, const CodeGenIntrinsic &Int) { + const FixedEncodingTy &FixedEncoding = FixedEncodings[Idx]; if ((Idx & 7) == 7) OS << "\n "; // If the entry fit in the table, just emit it. if ((FixedEncoding & Mask) == FixedEncoding) { OS << "0x" << Twine::utohexstr(FixedEncoding) << ", "; - continue; + return; } TypeSigTy TypeSig = ComputeTypeSignature(Int); unsigned Offset = LongEncodingTable.get(TypeSig); MaxOffset = std::max(MaxOffset, Offset); - // Otherwise, emit the offset into the long encoding table. We emit it this - // way so that it is easier to read the offset in the .def file. + // Otherwise, emit the offset into the long encoding table. We emit it + // this way so that it is easier to read the offset in the .def file. OS << formatv("(1U<<{}) | {}, ", MSBPostion, Offset); - } + }); OS << "0\n};\n\n"; @@ -485,7 +491,7 @@ static AttributeSet getIntrinsicArgAttributeSet(LLVMContext &C, unsigned ID) { // Compute unique argument attribute sets. std::map, unsigned> UniqArgAttributes; - for (const CodeGenIntrinsic &Int : Ints) { + Ints.enumerateIntrinsics([&](unsigned, const CodeGenIntrinsic &Int) { for (auto &Attrs : Int.ArgumentAttributes) { if (Attrs.empty()) continue; @@ -512,7 +518,7 @@ static AttributeSet getIntrinsicArgAttributeSet(LLVMContext &C, unsigned ID) { } OS << " });"; } - } + }); OS << R"( } } // getIntrinsicArgAttributeSet @@ -526,12 +532,12 @@ static AttributeSet getIntrinsicFnAttributeSet(LLVMContext &C, unsigned ID) { switch (ID) { default: llvm_unreachable("Invalid attribute set number");)"; - for (const CodeGenIntrinsic &Int : Ints) { + Ints.enumerateIntrinsics([&](unsigned, const CodeGenIntrinsic &Int) { if (!hasFnAttributes(Int)) - continue; + return; unsigned ID = UniqFnAttributes.size(); if (!UniqFnAttributes.try_emplace(&Int, ID).second) - continue; + return; OS << formatv(R"( case {}: return AttributeSet::get(C, {{ @@ -573,7 +579,7 @@ static AttributeSet getIntrinsicFnAttributeSet(LLVMContext &C, unsigned ID) { ME.toIntValue()); } OS << " });"; - } + }); OS << R"( } } // getIntrinsicFnAttributeSet @@ -586,10 +592,10 @@ AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id) { // arguments or not. std::map UniqAttributes; - for (const CodeGenIntrinsic &Int : Ints) { + Ints.enumerateIntrinsics([&](unsigned, const CodeGenIntrinsic &Int) { unsigned ID = UniqAttributes.size(); UniqAttributes.try_emplace(&Int, ID); - } + }); // Assign a 16-bit packed ID for each intrinsic. The lower 8-bits will be its // "argument attribute ID" (index in UniqAttributes) and upper 8 bits will be @@ -602,18 +608,20 @@ AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id) { // Emit an array of AttributeList. Most intrinsics will have at least one // entry, for the function itself (index ~1), which is usually nounwind. OS << " static constexpr uint16_t IntrinsicsToAttributesMap[] = {"; - for (const CodeGenIntrinsic &Int : Ints) { + Ints.enumerateIntrinsics([&](unsigned, const CodeGenIntrinsic &Int) { uint16_t FnAttrIndex = hasFnAttributes(Int) ? UniqFnAttributes[&Int] : 0; OS << formatv("\n {} << 8 | {}, // {}", FnAttrIndex, UniqAttributes[&Int], Int.Name); - } + }); OS << formatv(R"( }; - if (id == 0) + // First remap the Intrinsic::ID to intrinsic index. + unsigned Idx = getLinearIndex(id); + if (Idx == 0) return AttributeList(); - uint16_t PackedID = IntrinsicsToAttributesMap[id - 1]; + uint16_t PackedID = IntrinsicsToAttributesMap[Idx - 1]; uint8_t FnAttrID = PackedID >> 8; switch(PackedID & 0xFF) {{ default: llvm_unreachable("Invalid attribute number"); @@ -676,10 +684,10 @@ void IntrinsicEmitter::EmitIntrinsicToBuiltinMap( std::pair, std::optional>; std::map BuiltinMap; - for (const CodeGenIntrinsic &Int : Ints) { + Ints.enumerateIntrinsics([&](unsigned, const CodeGenIntrinsic &Int) { StringRef BuiltinName = IsClang ? Int.ClangBuiltinName : Int.MSBuiltinName; if (BuiltinName.empty()) - continue; + return; // Get the map for this target prefix. auto &[Map, CommonPrefix] = BuiltinMap[Int.TargetPrefix]; @@ -692,14 +700,14 @@ void IntrinsicEmitter::EmitIntrinsicToBuiltinMap( if (!CommonPrefix) { // For the first builtin for this target, initialize the common prefix. CommonPrefix = BuiltinName; - continue; + return; } // Update the common prefix. Note that this assumes that `take_front` will // never set the `Data` pointer in CommonPrefix to nullptr. const char *Mismatch = mismatch(*CommonPrefix, BuiltinName).first; *CommonPrefix = CommonPrefix->take_front(Mismatch - CommonPrefix->begin()); - } + }); // Populate the string table with the names of all the builtins after // removing this common prefix. @@ -727,7 +735,7 @@ Intrinsic::getIntrinsicFor{}Builtin(StringRef TargetPrefix, if (BuiltinMap.empty()) { OS << formatv(R"( return not_intrinsic; - } +} #endif // GET_LLVM_INTRINSIC_FOR_{}_BUILTIN )", UpperCompilerName); @@ -752,11 +760,13 @@ Intrinsic::getIntrinsicFor{}Builtin(StringRef TargetPrefix, // Emit a per target table of bultin names. bool HasTargetIndependentBuiltins = false; + bool HasTargetDependentBuiltins = false; StringRef TargetIndepndentCommonPrefix; for (const auto &[TargetPrefix, Entry] : BuiltinMap) { const auto &[Map, CommonPrefix] = Entry; if (!TargetPrefix.empty()) { OS << formatv(" // Builtins for {0}.\n", TargetPrefix); + HasTargetDependentBuiltins = true; } else { OS << " // Target independent builtins.\n"; HasTargetIndependentBuiltins = true; @@ -774,30 +784,32 @@ Intrinsic::getIntrinsicFor{}Builtin(StringRef TargetPrefix, OS << formatv(" }; // {}Names\n\n", TargetPrefix); } - // After emitting the builtin tables for all targets, emit a lookup table for - // all targets. We will use binary search, similar to the table for builtin - // names to lookup into this table. - OS << R"( - struct TargetEntry { - StringLiteral TargetPrefix; - ArrayRef Names; - StringLiteral CommonPrefix; - bool operator<(StringRef RHS) const { - return TargetPrefix < RHS; + if (HasTargetDependentBuiltins) { + // After emitting the builtin tables for all targets, emit a lookup table + // for all targets. We will use binary search, similar to the table for + // builtin names to lookup into this table. + OS << R"( + struct TargetEntry { + StringLiteral TargetPrefix; + ArrayRef Names; + StringLiteral CommonPrefix; + bool operator<(StringRef RHS) const { + return TargetPrefix < RHS; + }; }; - }; - static constexpr TargetEntry TargetTable[] = { -)"; + static constexpr TargetEntry TargetTable[] = { + )"; - for (const auto &[TargetPrefix, Entry] : BuiltinMap) { - const auto &[Map, CommonPrefix] = Entry; - if (TargetPrefix.empty()) - continue; - OS << formatv(R"( {{"{0}", {0}Names, "{1}"},)", TargetPrefix, - CommonPrefix) - << "\n"; + for (const auto &[TargetPrefix, Entry] : BuiltinMap) { + const auto &[Map, CommonPrefix] = Entry; + if (TargetPrefix.empty()) + continue; + OS << formatv(R"( {{"{0}", {0}Names, "{1}"},)", TargetPrefix, + CommonPrefix) + << "\n"; + } + OS << " };\n"; } - OS << " };\n"; // Now for the actual lookup, first check the target independent table if // we emitted one. @@ -816,8 +828,10 @@ Intrinsic::getIntrinsicFor{}Builtin(StringRef TargetPrefix, TargetIndepndentCommonPrefix); } - // If a target independent builtin was not found, lookup the target specific. - OS << formatv(R"( + if (HasTargetDependentBuiltins) { + // If a target independent builtin was not found, lookup the target + // specific. + OS << R"( auto TI = lower_bound(TargetTable, TargetPrefix); if (TI == std::end(TargetTable) || TI->TargetPrefix != TargetPrefix) return not_intrinsic; @@ -829,6 +843,12 @@ Intrinsic::getIntrinsicFor{}Builtin(StringRef TargetPrefix, if (II == std::end(TI->Names) || II->getName() != BuiltinName) return not_intrinsic; return II->IntrinsicID; +)"; + } else { + OS << " return not_intrinsic;"; + } + + OS << formatv(R"( } #endif // GET_LLVM_INTRINSIC_FOR_{}_BUILTIN